benchmark_plot

library(dplyr)
library(ggplot2)
library(pheatmap)
library(RColorBrewer)
library(reshape2)
library(ggpubr)
library(ComplexHeatmap)
library(circlize)
library(readxl)
library(plyr)
library(knitr)
library(stargazer)
setwd("~/Dropbox (Sydney Uni)/YunweiZhang/WeeklyProgress/R code/benchmarking/202104summaryresult")

• this outputs plots for manuscript

1 data

plot_fun=function(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1, coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit){
cox1dt=bind_rows(cox1, .id = "column_label")
cox1value=colMeans(cox1dt[,-1],na.rm = TRUE)
cox2dt=bind_rows(cox2, .id = "column_label")
cox2value=colMeans(cox2dt[,-1],na.rm = TRUE)
cox3dt=bind_rows(cox3, .id = "column_label")
cox3value=colMeans(cox3dt[,-1],na.rm = TRUE)
#this contains NA 

cox4=cox4[lapply(cox4,length)>1]
cox4dt=bind_rows(cox4, .id = "column_label")
cox4value=colMeans(cox4dt[,-1],na.rm = TRUE)
#cox5dt=bind_rows(cox5, .id = "column_label")
#cox5value=colMeans(cox5dt[,-1],na.rm = TRUE)
#cox6dt=bind_rows(cox6, .id = "column_label")
#cox6value=colMeans(cox6dt[,-1],na.rm = TRUE)
pcox1dt=bind_rows(pcox1, .id = "column_label")
pcox1value=colMeans(pcox1dt[,-1],na.rm = TRUE)
pcox2dt=bind_rows(pcox2, .id = "column_label")
pcox2value=colMeans(pcox2dt[,-1],na.rm = TRUE)
pcox3dt=bind_rows(pcox3, .id = "column_label")
pcox3value=colMeans(pcox3dt[,-1],na.rm = TRUE)
rsf1dt=bind_rows(rsf1, .id = "column_label")
rsf1value=colMeans(rsf1dt[,-1],na.rm = TRUE)
rsf2dt=bind_rows(rsf2, .id = "column_label")
rsf2value=colMeans(rsf2dt[,-1],na.rm = TRUE)
mtlr1dt=bind_rows(mtlr1, .id = "column_label")
mtlr1value=colMeans(mtlr1dt[,-1],na.rm = TRUE)
dnnsurv1dt=bind_rows(dnnsurv1, .id = "column_label")
dnnsurv1value=colMeans(dnnsurv1dt[,-1],na.rm = TRUE)
coxboostdt=bind_rows(coxboost, .id = "column_label")
coxboostvalue=colMeans(coxboostdt[,-1],na.rm = TRUE)
gacoxdt=bind_rows(gacox, .id = "column_label")
gacoxvalue=colMeans(gacoxdt[,-1],na.rm = TRUE)
gamtlrdt=bind_rows(gamtlr, .id = "column_label")
gamtlrvalue=colMeans(gamtlrdt[,-1],na.rm = TRUE)
gacoxboostdt=bind_rows(gacoxboost, .id = "column_label")
gacoxboostvalue=colMeans(gacoxboostdt[,-1],na.rm = TRUE)
limmamtlrdt=bind_rows(limmamtlr, .id = "column_label")
limmamtlrvalue=colMeans(limmamtlrdt[,-1],na.rm = TRUE)
limmacoxboostdt=bind_rows(limmacoxboost, .id = "column_label")
limmacoxboostvalue=colMeans(limmacoxboostdt[,-1],na.rm = TRUE)
survivalsvmdt=bind_rows(survivalsvm, .id = "column_label")
survivalsvmvalue=colMeans(survivalsvmdt[,-1],na.rm = TRUE)
deepsurvdt=bind_rows(deepsurv, .id = "column_label")
deepsurvvalue=colMeans(deepsurvdt[,-1],na.rm = TRUE)
deephitdt=bind_rows(deephit, .id = "column_label")
deephitvalue=colMeans(deephitdt[,-1],na.rm = TRUE)





nb.cols <- 20
mycolors <- colorRampPalette(brewer.pal(8, "Set2"))(nb.cols)

plotdt1=rbind.data.frame(cox1value,cox2value,cox3value,cox4value,pcox1value,pcox2value,pcox3value,rsf1value,rsf2value,mtlr1value,dnnsurv1value,coxboostvalue,gacoxvalue,gamtlrvalue,gacoxboostvalue,limmamtlrvalue,limmacoxboostvalue,survivalsvmvalue,deepsurvvalue,deephitvalue)
rownames(plotdt1)=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
colnames(plotdt1)=names(pcox1value)
head(plotdt1)
plotdt2=t(plotdt1)
#head(plotdt2)
#plotdt3=apply(plotdt2, MARGIN = 2, FUN = function(X) (X - min(X))/diff(range(X)))
plotdt3=plotdt2[-c(2,4,5:10),]
#plotdt3[plotdt3>1]=NA #out of range (0,1) is definde as NA
breaksList = seq(0.5, 1, by = 0.1)
#pheatmap(as.matrix(plotdt3),cellwidth = 15, cellheight = 15, fontsize = 8,cluster_rows = FALSE,cluster_cols = FALSE,color =colorRampPalette(c("#999999", "#E69F00", "#56B4E9"))(length(breaksList)),breaks = breaksList)
p1=pheatmap(as.matrix(plotdt3),cellwidth = 15, cellheight = 15, fontsize = 8,cluster_rows = FALSE,cluster_cols = FALSE,color =colorRampPalette(rev(brewer.pal(n = 7, name = "Pastel1")))(length(breaksList)),breaks = breaksList)
#print(p1)

# create data frame with all values
colnames(cox1dt)=colnames(cox2dt)=colnames(cox3dt)=colnames(cox4dt)=colnames(pcox1dt)=colnames(pcox2dt)=colnames(pcox3dt)=colnames(rsf1dt)=colnames(rsf2dt)=colnames(mtlr1dt)=colnames(dnnsurv1dt)=colnames(coxboostdt)=colnames(gacoxdt)=colnames(gamtlrdt)=colnames(gacoxboostdt)=colnames(limmamtlrdt)=colnames(limmacoxboostdt)=colnames(survivalsvmdt)=colnames(deepsurvdt)=colnames(deephitdt)=c("model","hc","bc","unoc","ghc","bs1","bs2","bs3","bs4","bs5","bs6","auc1","auc2","auc3","auc4","auc5","auc6","auc7","auc8","auc9","auc10","auc11","auc12","auc13","auc14","auc15","auc")
widedt=do.call("rbind", list(cox1dt,cox2dt,cox3dt,cox4dt,pcox1dt,pcox2dt,pcox3dt,rsf1dt,rsf2dt,mtlr1dt,dnnsurv1dt,coxboostdt,gacoxdt,gamtlrdt,gacoxboostdt,limmamtlrdt,limmacoxboostdt,survivalsvmdt,deepsurvdt,deephitdt))
widedt[,1]=c(rep("cox1",nrow(cox1dt)),rep("cox2",nrow(cox2dt)),rep("cox3",nrow(cox3dt)),rep("cox4",nrow(cox4dt)),rep("pcox1",nrow(pcox1dt)),rep("pcox2",nrow(pcox2dt)),rep("pcox3",nrow(pcox3dt)),rep("rsf1",nrow(rsf1dt)),rep("rsf2",nrow(rsf2dt)),rep("mtlr1",nrow(mtlr1dt)),rep("dnnsurv1",nrow(dnnsurv1dt)),rep("coxboost",nrow(coxboostdt)),rep("gacox",nrow(gacoxdt)),rep("gamtlr",nrow(gamtlrdt)),rep("gacoxboost",nrow(gacoxboostdt)),rep("limmamtlr",nrow(limmamtlrdt)),rep("limmacoxboost",nrow(limmacoxboostdt)),rep("survivalsvm",nrow(survivalsvmdt)),rep("deepsurv",nrow(deepsurvdt)),rep("deephit",nrow(deephitdt)))
# grouped boxplot
longdt=melt(widedt,id.vars = "model",na.rm = TRUE)
#dim(longdt)
#any(is.na(longdt))

# get the hc
plotdt=longdt[longdt$variable=="hc",]
#plotdt[plotdt$value <0.5 | plotdt$value >0.99,3] <- NA
#plotdt[plotdt$value < quantile(plotdt$value, 0.85,na.rm = T) | plotdt$value > quantile(plotdt$value, 0.15,na.rm = T), ]

plotdt$model=factor(plotdt$model,levels=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit"))
g1=ggplot(plotdt, aes(x=model, y=value, fill=model)) + geom_boxplot(lwd=0.5)+scale_fill_manual(values = mycolors,drop=FALSE)+theme_bw() #+scale_y_continuous(limits=c(0.5,1))

# get all the cindex
plotdt=longdt[longdt$variable=="hc"|longdt$variable=="bc"|longdt$variable=="unoc"|longdt$variable=="ghc",]
plotdt$model=factor(plotdt$model,levels=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit"))
g2=ggplot(plotdt, aes(x=model, y=value, fill=variable)) + geom_boxplot(lwd=0.5)+scale_fill_manual(values = mycolors)+theme_bw() #+scale_y_continuous(limits=c(0.5,1))

# get all the auc
plotdt=longdt[longdt$variable=="auc1"|longdt$variable=="auc2"|longdt$variable=="auc3"|longdt$variable=="auc4",]
plotdt$model=factor(plotdt$model,levels=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit"))
g3=ggplot(plotdt, aes(x=model, y=value, fill=variable)) + geom_boxplot(lwd=0.5)+scale_fill_manual(values = mycolors)+theme_bw() #+scale_y_continuous(limits=c(0.5,1))

# get all the bs
plotdt=longdt[longdt$variable=="bs1"|longdt$variable=="bs2",]
plotdt$model=factor(plotdt$model,levels=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit"))
g4=ggplot(plotdt, aes(x=model, y=value, fill=variable)) + geom_boxplot(lwd=0.1,outlier.shape = NA)+scale_fill_manual(values = mycolors) +theme_bw()

# get bs1
plotdt=longdt[longdt$variable=="bs1",]
plotdt$model=factor(plotdt$model,levels=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit"))
g5=ggplot(plotdt, aes(x=model, y=value, fill=model)) + geom_boxplot(lwd=0.5,outlier.shape = NA)+scale_fill_manual(values = mycolors, drop=FALSE)+theme_bw()



# get auc curve
plotdt=longdt[longdt$variable=="auc1"|longdt$variable=="auc2"|longdt$variable=="auc3"|longdt$variable=="auc4"|longdt$variable=="auc5"|longdt$variable=="auc6"|longdt$variable=="auc7"|longdt$variable=="auc8"|longdt$variable=="auc9"|longdt$variable=="auc10"|longdt$variable=="auc11"|longdt$variable=="auc12"|longdt$variable=="auc13"|longdt$variable=="auc14"|longdt$variable=="auc15",]
#plotdt[plotdt$value >1 | plotdt$value==0,3] <- NA

plotdt2 <- plotdt %>%dplyr::group_by(variable,model) %>%dplyr::summarize(Mean = mean(value))
plotdt2$timepoint=as.numeric(gsub(".*?([0-9]+).*", "\\1", plotdt2$variable))
# g6=ggplot(data = plotdt2, aes(timepoint, Mean)) +
#   geom_line() +geom_smooth()+
#   labs(title = "                                                            time-dependent AUC",
#        y = "value", x = "time points") + 
#   facet_wrap(~ model)#+scale_y_continuous(limits=c(0,1))
plotdt2$model=factor(plotdt2$model,levels=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit"))
names(mycolors)=levels(plotdt2$model)
g6=ggplot(plotdt2,aes(x=timepoint,y=Mean,color=model))+geom_line(aes(color=model))+scale_color_manual(values = mycolors, drop=FALSE)+theme_bw()
# print(g1)
# print(g2)
# print(g3)
# print(g4)
# print(g5)
# print(g6)
#print(ggarrange(g1,g2, g3,g4,g5,labels = c("c","c", "auc", "br","ibr"),ncol = 2, nrow = 3))
return(list(longdt,g1,g2,g3,g4,g5,g6,p1))
}

characteristic_table=function(cox4){
cox4dt=bind_rows(cox4, .id = "column_label")
colnames(cox4dt)=c("model","hc","bc","unoc","ghc","bs1","bs2","bs3","bs4","bs5","bs6","auc1","auc2","auc3","auc4","auc5","auc6","auc7","auc8","auc9","auc10","auc11","auc12","auc13","auc14","auc15","auc")
proportion=NULL
summaryy=NULL
if(sum(rowSums(is.na(cox4dt))==26)!=0){
  proportion=1-sum(rowSums(is.na(cox4dt))==26)/100
cox4dt=cox4dt[-which(rowSums(is.na(cox4dt))==26),]
summaryy=summary(cox4dt)}else{
  proportion=1
  summaryy=summary(cox4dt)
}
return(list(proportion,summaryy))
}

1.1 pbc

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=readRDS("savedresults/pbc_cox1.rds")
cox2=readRDS("savedresults/pbc_bw_cox1.rds")
cox3=readRDS("savedresults/pbc_bw_cox2.rds")
cox4=readRDS("savedresults/pbc_bw_cox3.rds")
pcox1=readRDS("savedresults/pbc_p_cox1.rds")
pcox2=readRDS("savedresults/pbc_p_cox2.rds")
pcox3=readRDS("savedresults/pbc_p_cox3.rds")
rsf1=readRDS("savedresults/pbc_rsf1.rds")
rsf2=readRDS("savedresults/pbc_rsf2.rds")
mtlr1=readRDS("savedresults/pbc_mtlr.rds")
dnnsurv1=readRDS("savedresults/pbc_dnnsurv.rds")
coxboost=readRDS("savedresults/pbc_coxboost.rds")
gacox=readRDS("savedresults/pbc_ga_cox1.rds")
gamtlr=readRDS("savedresults/pbc_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/pbc_ga_coxboost.rds")
limmamtlr=readRDS("savedresults/pbc_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/pbc_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/pbc_survivalsvm.rds")
deepsurv=readRDS("savedresults/pbc_deepsurv.rds")
deephit=readRDS("savedresults/pbc_deephit.rds")
want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

pbcfull=want[[1]]
g1=want[[2]]
g2=want[[3]]
g3=want[[4]]
g4=want[[5]]
g5=want[[6]]
g6=want[[7]]
g7=want[[8]]
characteristic_table(dnnsurv1)[[1]]

## [1] 0.8

#ggsave(plot=g6,file="pbc_auc.pdf",device = "pdf")

1.2 veteran

cox1=readRDS("savedresults/veteran_cox1.rds")
cox2=readRDS("savedresults/veteran_bw_cox1.rds")
cox3=readRDS("savedresults/veteran_bw_cox2.rds")
cox4=readRDS("savedresults/veteran_bw_cox3.rds")
pcox1=readRDS("savedresults/veteran_p_cox1.rds")
pcox2=readRDS("savedresults/veteran_p_cox2.rds")
pcox3=readRDS("savedresults/veteran_p_cox3.rds")
rsf1=readRDS("savedresults/veteran_rsf1.rds")
rsf2=readRDS("savedresults/veteran_rsf2.rds")
mtlr1=readRDS("savedresults/veteran_mtlr.rds")
dnnsurv1=readRDS("savedresults/veteran_dnnsurv.rds")
coxboost=readRDS("savedresults/veteran_coxboost.rds")
gacox=readRDS("savedresults/veteran_ga_cox1.rds")
summ=0
 for (i in 1:length(gacox)){
    if (class(gacox[[i]])=="try-error"){
      summ=summ+1
    gacox[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{gacox[[i]]=gacox[[i]]}
    colnames(gacox[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 3

gamtlr=readRDS("savedresults/veteran_ga_mtlr.rds")

gacoxboost=readRDS("savedresults/veteran_ga_coxboost.rds")

limmamtlr=readRDS("savedresults/veteran_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/veteran_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/veteran_survivalsvm.rds")
deepsurv=readRDS("savedresults/veteran_deepsurv.rds")
deephit=readRDS("savedresults/veteran_deephit.rds")
want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

veteranfull=want[[1]]
g8=want[[2]]
g9=want[[3]]
g10=want[[4]]
g11=want[[5]]
g12=want[[6]]
g13=want[[7]]
g14=want[[8]]

characteristic_table(dnnsurv1)[[1]]

## [1] 0.66

characteristic_table(gacox)[[1]]

## [1] 0.85

1.3 lung

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=readRDS("savedresults/lung_cox1.rds")
cox2=readRDS("savedresults/lung_bw_cox1.rds")
cox3=readRDS("savedresults/lung_bw_cox2.rds")
cox4=readRDS("savedresults/lung_bw_cox3.rds")
pcox1=readRDS("savedresults/lung_p_cox1.rds")
pcox2=readRDS("savedresults/lung_p_cox2.rds")
pcox3=readRDS("savedresults/lung_p_cox3.rds")
rsf1=readRDS("savedresults/lung_rsf1.rds")
rsf2=readRDS("savedresults/lung_rsf2.rds")
mtlr1=readRDS("savedresults/lung_mtlr.rds")
dnnsurv1=readRDS("savedresults/lung_dnnsurv.rds")
coxboost=readRDS("savedresults/lung_coxboost.rds")
gacox=readRDS("savedresults/lung_ga_cox1.rds")
gamtlr=readRDS("savedresults/lung_ga_mtlr.rds")

gacoxboost=readRDS("savedresults/lung_ga_coxboost.rds")

limmamtlr=readRDS("savedresults/lung_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/lung_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/lung_survivalsvm.rds")
deepsurv=readRDS("savedresults/lung_deepsurv.rds")
deephit=readRDS("savedresults/lung_deephit.rds")
want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

lungfull=want[[1]]
g15=want[[2]]
g16=want[[3]]
g17=want[[4]]
g18=want[[5]]
g19=want[[6]]
g20=want[[7]]
g21=want[[8]]
#ggsave(plot=g20,file="lung_auc.pdf",device = "pdf")
characteristic_table(dnnsurv1)[[1]]

## [1] 1

1.4 ANZ

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=readRDS("savedresults/anz_cox1.rds")
cox2=readRDS("savedresults/anz_bw_cox1.rds")
cox3=readRDS("savedresults/anz_bw_cox2.rds")
cox4=readRDS("savedresults/anz_bw_cox3.rds")
pcox1=readRDS("savedresults/anz_p_cox1.rds")
pcox2=readRDS("savedresults/anz_p_cox2.rds")
pcox3=readRDS("savedresults/anz_p_cox3.rds")
rsf1=readRDS("savedresults/anz_rsf1.rds")
rsf2=readRDS("savedresults/anz_rsf2.rds")
mtlr1=readRDS("savedresults/anz_mtlr.rds")
dnnsurv1=readRDS("savedresults/anz_dnnsurv.rds")
coxboost=readRDS("savedresults/anz_coxboost.rds")
gacox=readRDS("savedresults/anz_ga_cox1.rds")
gamtlr=readRDS("savedresults/anz_ga_mtlr.rds")

gacoxboost=readRDS("savedresults/anz_ga_coxboost.rds")

limmamtlr=readRDS("savedresults/anz_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/anz_limma_coxboost.rds")
survivalsvm=empty_list
deepsurv=readRDS("savedresults/anz_deepsurv.rds")
deephit=readRDS("savedresults/anz_deephit.rds")
want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

anzfull=want[[1]]
g22=want[[2]]
g23=want[[3]]
g24=want[[4]]
g25=want[[5]]
g26=want[[6]]
g27=want[[7]]
g28=want[[8]]



characteristic_table(dnnsurv1)[[1]]

## [1] 1

#characteristic_table(pcox3)

1.5 US

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=readRDS("savedresults/us_cox1.rds")
cox2=empty_list
cox3=empty_list
cox4=empty_list
#pcox1=readRDS("US/cox4.rds")
 # for (i in 1:length(dnnsurv1)){
 #    if (class(dnnsurv1[[i]])=="try-error")
 #    dnnsurv1[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
 #    colnames(dnnsurv1[[i]])=c("hc", "bc", "unoc" ,"ghc" , "br1" , "br2" ,"br3","br4",  "br5", "br6" , "a1"  , "a2","a3" , "a4","a5" , "a6", "a7", "a8", "a9", "a10" , "a11" , "a12" , "a13",  "a14" , "a15" , "a")}
pcox1=readRDS("savedresults/us_p_cox1.rds")
pcox2=readRDS("savedresults/us_p_cox2.rds")
pcox3=readRDS("savedresults/us_p_cox3.rds")
rsf1=readRDS("savedresults/us_rsf1.rds")
rsf2=readRDS("savedresults/us_rsf2.rds")
mtlr1=readRDS("savedresults/us_mtlr.rds")
dnnsurv1=empty_list
coxboost=readRDS("savedresults/us_coxboost.rds")
gacox=empty_list
gamtlr=empty_list
gacoxboost=empty_list
limmamtlr=readRDS("savedresults/us_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/us_limma_coxboost.rds")
survivalsvm=empty_list
deepsurv=readRDS("savedresults/us_deepsurv.rds")
deephit=readRDS("savedresults/us_deephit.rds")
want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

usfull=want[[1]]
g29=want[[2]]
g30=want[[3]]
g31=want[[4]]
g32=want[[5]]
g33=want[[6]]
g34=want[[7]]
g35=want[[8]]
#ggsave(plot=g34,file="us_auc.pdf",device = "pdf")
#characteristic_table(rsf1)[[1]]
characteristic_table(pcox3)[[1]]

## [1] 1

1.6 melanoma_clinical

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=readRDS("savedresults/melanomaclinical_cox1.rds")
cox2=empty_list
cox3=empty_list
cox4=empty_list
#pcox1=readRDS("US/cox4.rds")
pcox1=readRDS("savedresults/melanomaclinical_p_cox1.rds")
pcox2=readRDS("savedresults/melanomaclinical_p_cox2.rds")
pcox3=readRDS("savedresults/melanomaclinical_p_cox3.rds")
rsf1=readRDS("savedresults/melanomaclinical_rsf1.rds")
rsf2=readRDS("savedresults/melanomaclinical_rsf2.rds")
mtlr1=readRDS("savedresults/melanomaclinical_mtlr.rds")
#dnnsurv1=readRDS("savedresults/melanomaclinical_dnnsurv.rds")
dnnsurv1=empty_list
# summ=0
#  for (i in 1:length(dnnsurv1)){
#     if (class(dnnsurv1[[i]])=="try-error"){
#       summ=summ+1
#     dnnsurv1[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{cox2[[i]]=cox2[[i]]}
#     colnames(dnnsurv1[[i]])=c("hc", "bc", "unoc" ,"ghc" , "br1" , "br2" ,"br3","br4",  "br5", "br6" , "a1"  , "a2","a3" , "a4","a5" , "a6", "a7", "a8", "a9", "a10" , "a11" , "a12" , "a13",  "a14" , "a15" , "a")}
# print(summ)

coxboost=readRDS("savedresults/melanomaclinica_coxboost.rds")
gacox=readRDS("savedresults/melanomaclinical_ga_cox1.rds")
gamtlr=readRDS("savedresults/melanomaclinical_ga_mtlr.rds")

gacoxboost=readRDS("savedresults/melanomaclinical_ga_coxboost.rds")
limmamtlr=readRDS("savedresults/melanomaclinical_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/melanomaclinical_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/clinical_survivalsvm.rds")
deepsurv=readRDS("savedresults/melanomaclinical_deepsurv.rds")
deephit=readRDS("savedresults/melanomaclinical_deephit.rds")
want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

melanoma_clinicalfull=want[[1]]
g36=want[[2]]
g37=want[[3]]
g38=want[[4]]
g39=want[[5]]
g40=want[[6]]
g41=want[[7]]
g42=want[[8]]

#characteristic_table(dnnsurv1)[[1]]

1.7 melanoma_itraq

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=empty_list
cox2=empty_list
cox3=empty_list
cox4=empty_list
# cox2=readRDS("melanomaitraq2/melanomaitraq_bw_cox1.rds")
# summ=0
#  for (i in 1:length(cox2)){
#     if (class(cox2[[i]])=="try-error"){
#       summ=summ+1
#     cox2[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{cox2[[i]]=cox2[[i]]}
#     colnames(cox2[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
# print(summ)
# cox3=readRDS("melanomaitraq2/melanomaitraq_bw_cox2.rds")
# summ=0
#  for (i in 1:length(cox3)){
#     if (class(cox3[[i]])=="try-error"){
#       summ=summ+1
#     cox3[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{cox2[[i]]=cox2[[i]]}
#     colnames(cox3[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
# print(summ)
# cox4=readRDS("melanomaitraq2/melanomaitraq_bw_cox3.rds")
# summ=0
#  for (i in 1:length(cox4)){
#     if (class(cox4[[i]])=="try-error"){
#       summ=summ+1
#     cox4[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{cox2[[i]]=cox2[[i]]}
#     colnames(cox4[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
# print(summ)
#pcox1=readRDS("US/cox4.rds")
pcox1=readRDS("savedresults/melanomaitraqv2_p_cox1.rds")
pcox2=empty_list
pcox3=empty_list
rsf1=readRDS("savedresults/melanomaitraqv2_rsf3.rds")
rsf2=readRDS("savedresults/melanomaitraqv2_rsf4.rds")
#mtlr1=readRDS("melanomaitraq2/melanomaitraq_mtlr.rds")
#can no longer run
mtlr1=empty_list
dnnsurv1=readRDS("savedresults/melanomaitraqv2_dnnsurv.rds")

# rsf1=rsf1$value
# rsf2=rsf2$value
# mtlr1=mtlr1$value
coxboost=readRDS("savedresults/melanomaitraqv2_coxboost.rds")
gacox=readRDS("savedresults/itraqv2_ga_cox1.rds")
gamtlr=readRDS("savedresults/melanomaitraqv2_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/melanomaitraqv2_ga_coxboost.rds")
limmamtlr=readRDS("savedresults/melanomaitraqv2_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/melanomaitraqv2_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/itraqv2_survivalsvm.rds")
deepsurv=readRDS("savedresults/melanomaitraqv2_deepsurv.rds")
deephit=readRDS("savedresults/melanomaitraqv2_deephit.rds")
want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

melanoma_itraqfull=want[[1]]
g43=want[[2]]
g44=want[[3]]
g45=want[[4]]
g46=want[[5]]
g47=want[[6]]
g48=want[[7]]
g49=want[[8]]
#ggsave(plot=g48,file="itraq_auc.pdf",device = "pdf")
#characteristic_table(rsf1)[[2]]
characteristic_table(dnnsurv1)[[1]]

## [1] 0.63

1.8 melanomanano

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=empty_list
cox2=readRDS("savedresults/melanomananov3_bw_cox1.rds")
summ=0
 for (i in 1:length(cox2)){
    if (class(cox2[[i]])=="try-error"){
      summ=summ+1
    cox2[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{cox2[[i]]=cox2[[i]]}
    colnames(cox2[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 16

cox3=readRDS("savedresults/melanomananov3_bw_cox2.rds")
summ=0
 for (i in 1:length(cox3)){
    if (class(cox3[[i]])=="try-error"){
      summ=summ+1
    cox3[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{cox2[[i]]=cox2[[i]]}
    colnames(cox3[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 20

cox4=readRDS("savedresults/melanomananov3_bw_cox3.rds")
summ=0
 for (i in 1:length(cox4)){
    if (class(cox4[[i]])=="try-error"){
      summ=summ+1
    cox4[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{cox2[[i]]=cox2[[i]]}
    colnames(cox4[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 16

pcox1=readRDS("savedresults/melanomananov3_p_cox1.rds")
summ=0
 for (i in 1:length(pcox1)){
    if (class(pcox1[[i]])=="try-error"){
      summ=summ+1
    pcox1[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{pcox1[[i]]=pcox1[[i]]}
    colnames(pcox1[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 0

pcox2=readRDS("savedresults/melanomananov3_p_cox2.rds")
summ=0
 for (i in 1:length(pcox2)){
    if (class(pcox2[[i]])=="try-error"){
      summ=summ+1
    pcox2[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{pcox2[[i]]=pcox2[[i]]}
    colnames(pcox2[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 0

pcox3=readRDS("savedresults/melanomananov3_p_cox3.rds")
summ=0
 for (i in 1:length(pcox3)){
    if (class(pcox3[[i]])=="try-error"){
      summ=summ+1
    pcox3[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{pcox3[[i]]=pcox3[[i]]}
    colnames(pcox3[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 0

rsf1=readRDS("savedresults/melanomananov3_rsf3.rds")
summ=0
 for (i in 1:length(rsf1)){
    if (class(rsf1[[i]])=="try-error"){
      summ=summ+1
    rsf1[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{rsf1[[i]]=rsf1[[i]]}
    colnames(rsf1[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 0

rsf2=readRDS("savedresults/melanomananov3_rsf4.rds")
summ=0
 for (i in 1:length(rsf2)){
    if (class(rsf2[[i]])=="try-error"){
      summ=summ+1
    rsf2[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{rsf2[[i]]=rsf2[[i]]}
    colnames(rsf2[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 0

mtlr1=readRDS("savedresults/melanomananov3_mtlr.rds")
summ=0
 for (i in 1:length(mtlr1)){
    if (class(mtlr1[[i]])=="try-error"){
      summ=summ+1
    mtlr1[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{mtlr1[[i]]=mtlr1[[i]]}
    colnames(mtlr1[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 0

dnnsurv1=readRDS("savedresults/melanomananov3_dnnsurv.rds")
summ=0
 for (i in 1:length(dnnsurv1)){
    if (class(dnnsurv1[[i]])=="try-error"){
      summ=summ+1
    dnnsurv1[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{dnnsurv1[[i]]=dnnsurv1[[i]]}
    colnames(dnnsurv1[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 0

coxboost=readRDS("savedresults/melanomananov3_coxboost.rds")
summ=0
 for (i in 1:length(coxboost)){
    if (class(coxboost[[i]])=="try-error"){
      summ=summ+1
    coxboost[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))}else{coxboost[[i]]=coxboost[[i]]}
    colnames(coxboost[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")}
print(summ)

## [1] 0

gacox=readRDS("savedresults/melanomananov3_ga_cox1.rds")
gamtlr=readRDS("savedresults/melanomananov3_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/melanomananov3_ga_coxboost.rds")
limmamtlr=readRDS("savedresults/melanomananov3_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/melanomananov3_limma_coxboost.rds")
deepsurv=readRDS("savedresults/melanomananov3_deepsurv.rds")
deephit=readRDS("savedresults/melanomananov3_deephit.rds")

survivalsvm=empty_list
want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

melanomananofull=want[[1]]
g57=want[[2]]
g58=want[[3]]
g59=want[[4]]
g60=want[[5]]
g61=want[[6]]
g62=want[[7]]
g63=want[[8]]

characteristic_table(rsf1)[[2]]

##     model                 hc               bc           unoc       
##  Length:100         Min.   :0.2121   Min.   : NA   Min.   :0.2121  
##  Class :character   1st Qu.:0.5942   1st Qu.: NA   1st Qu.:0.5942  
##  Mode  :character   Median :0.6970   Median : NA   Median :0.6985  
##                     Mean   :0.6831   Mean   :NaN   Mean   :0.6851  
##                     3rd Qu.:0.7631   3rd Qu.: NA   3rd Qu.:0.7673  
##                     Max.   :1.0000   Max.   : NA   Max.   :1.0000  
##                                      NA's   :100                   
##       ghc           bs1             bs2             bs3        
##  Min.   : NA   Min.   :0.869   Min.   :0.869   Min.   :0.1046  
##  1st Qu.: NA   1st Qu.:1.793   1st Qu.:1.793   1st Qu.:0.1441  
##  Median : NA   Median :2.179   Median :2.179   Median :0.1819  
##  Mean   :NaN   Mean   :2.340   Mean   :2.340   Mean   :   Inf  
##  3rd Qu.: NA   3rd Qu.:2.610   3rd Qu.:2.610   3rd Qu.:0.5065  
##  Max.   : NA   Max.   :7.385   Max.   :7.385   Max.   :   Inf  
##  NA's   :100                                   NA's   :23      
##       bs4              bs5              bs6              auc1        
##  Min.   :0.1061   Min.   :0.1061   Min.   :0.1061   Min.   :0.05556  
##  1st Qu.:0.1628   1st Qu.:0.1628   1st Qu.:0.1628   1st Qu.:0.50000  
##  Median :0.1949   Median :0.1949   Median :0.1949   Median :0.63393  
##  Mean   :0.1987   Mean   :0.1987   Mean   :0.1987   Mean   :0.63224  
##  3rd Qu.:0.2193   3rd Qu.:0.2193   3rd Qu.:0.2193   3rd Qu.:0.78571  
##  Max.   :0.5065   Max.   :0.5065   Max.   :0.5065   Max.   :1.00000  
##                                                                      
##       auc2             auc3             auc4             auc5       
##  Min.   :0.1500   Min.   :0.1500   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.6083   1st Qu.:0.7500   1st Qu.:0.7000   1st Qu.:0.7000  
##  Median :0.7361   Median :0.8500   Median :0.8000   Median :0.8000  
##  Mean   :0.7245   Mean   :0.8317   Mean   :0.7881   Mean   :0.7881  
##  3rd Qu.:0.8500   3rd Qu.:0.9458   3rd Qu.:0.9071   3rd Qu.:0.9071  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##                                                                     
##       auc6             auc7             auc8             auc9       
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.7000   1st Qu.:0.7000   1st Qu.:0.6667   1st Qu.:0.6823  
##  Median :0.8000   Median :0.8000   Median :0.8000   Median :0.8500  
##  Mean   :0.7881   Mean   :0.7881   Mean   :0.7792   Mean   :0.7970  
##  3rd Qu.:0.9071   3rd Qu.:0.9071   3rd Qu.:0.9071   3rd Qu.:0.9444  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##                                                                     
##      auc10            auc11            auc12            auc13       
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.6667   1st Qu.:0.6667   1st Qu.:0.7333   1st Qu.:0.7333  
##  Median :0.8417   Median :0.8333   Median :0.8593   Median :0.8593  
##  Mean   :0.7954   Mean   :0.7820   Mean   :0.8087   Mean   :0.8087  
##  3rd Qu.:0.9392   3rd Qu.:0.9375   3rd Qu.:1.0000   3rd Qu.:1.0000  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##                                                                     
##      auc14            auc15             auc        
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.1479  
##  1st Qu.:0.7333   1st Qu.:0.5958   1st Qu.:0.6398  
##  Median :0.8593   Median :0.7955   Median :0.7379  
##  Mean   :0.8087   Mean   :0.7151   Mean   :0.7246  
##  3rd Qu.:1.0000   3rd Qu.:0.9392   3rd Qu.:0.8197  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
## 

characteristic_table(pcox1)[[1]]

## [1] 1

characteristic_table(pcox1)[[2]]

##     model                 hc               bc              unoc       
##  Length:100         Min.   :0.2121   Min.   :0.2361   Min.   :0.0000  
##  Class :character   1st Qu.:0.5000   1st Qu.:0.4933   1st Qu.:0.5000  
##  Mode  :character   Median :0.5917   Median :0.5694   Median :0.6000  
##                     Mean   :0.5904   Mean   :0.5725   Mean   :0.5871  
##                     3rd Qu.:0.6667   3rd Qu.:0.6528   3rd Qu.:0.6676  
##                     Max.   :0.9615   Max.   :1.0000   Max.   :0.9615  
##                                                                       
##       ghc              bs1             bs2             bs3           bs4     
##  Min.   :0.0000   Min.   :0.816   Min.   :0.816   Min.   : NA   Min.   : NA  
##  1st Qu.:0.6086   1st Qu.:1.847   1st Qu.:1.847   1st Qu.: NA   1st Qu.: NA  
##  Median :0.6997   Median :2.097   Median :2.097   Median : NA   Median : NA  
##  Mean   :0.7122   Mean   :2.251   Mean   :2.251   Mean   :NaN   Mean   :NaN  
##  3rd Qu.:0.8111   3rd Qu.:2.586   3rd Qu.:2.586   3rd Qu.: NA   3rd Qu.: NA  
##  Max.   :0.9653   Max.   :6.249   Max.   :6.249   Max.   : NA   Max.   : NA  
##                                                   NA's   :100   NA's   :100  
##       bs5           bs6           auc1             auc2             auc3       
##  Min.   : NA   Min.   : NA   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.: NA   1st Qu.: NA   1st Qu.:0.3889   1st Qu.:0.3889   1st Qu.:0.5000  
##  Median : NA   Median : NA   Median :0.5556   Median :0.5528   Median :0.6500  
##  Mean   :NaN   Mean   :NaN   Mean   :0.5521   Mean   :0.5520   Mean   :0.6383  
##  3rd Qu.: NA   3rd Qu.: NA   3rd Qu.:0.7500   3rd Qu.:0.6875   3rd Qu.:0.8000  
##  Max.   : NA   Max.   : NA   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##  NA's   :100   NA's   :100                                                     
##       auc4             auc5             auc6             auc7       
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.5000   1st Qu.:0.5000   1st Qu.:0.5000   1st Qu.:0.5000  
##  Median :0.6500   Median :0.6500   Median :0.6500   Median :0.6500  
##  Mean   :0.6361   Mean   :0.6361   Mean   :0.6361   Mean   :0.6361  
##  3rd Qu.:0.7893   3rd Qu.:0.7893   3rd Qu.:0.7893   3rd Qu.:0.7893  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##                                                                     
##       auc8             auc9            auc10            auc11       
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.5000   1st Qu.:0.5000   1st Qu.:0.4594   1st Qu.:0.4286  
##  Median :0.6111   Median :0.6000   Median :0.6056   Median :0.6000  
##  Mean   :0.6186   Mean   :0.6037   Mean   :0.6052   Mean   :0.5931  
##  3rd Qu.:0.7857   3rd Qu.:0.7857   3rd Qu.:0.7857   3rd Qu.:0.7857  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##                                                                     
##      auc12            auc13            auc14            auc15       
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.4667   1st Qu.:0.4667   1st Qu.:0.4667   1st Qu.:0.3989  
##  Median :0.6000   Median :0.6000   Median :0.6000   Median :0.5328  
##  Mean   :0.5964   Mean   :0.5964   Mean   :0.5964   Mean   :0.5319  
##  3rd Qu.:0.7589   3rd Qu.:0.7589   3rd Qu.:0.7589   3rd Qu.:0.7500  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##                                                                     
##       auc        
##  Min.   :0.1429  
##  1st Qu.:0.4679  
##  Median :0.5833  
##  Mean   :0.5896  
##  3rd Qu.:0.7108  
##  Max.   :1.0000  
## 

characteristic_table(pcox3)[[2]]

##     model                 hc               bc              unoc       
##  Length:100         Min.   :0.2121   Min.   :0.2361   Min.   :0.2121  
##  Class :character   1st Qu.:0.4803   1st Qu.:0.4715   1st Qu.:0.4803  
##  Mode  :character   Median :0.6030   Median :0.5833   Median :0.6089  
##                     Mean   :0.5985   Mean   :0.5765   Mean   :0.5992  
##                     3rd Qu.:0.7000   3rd Qu.:0.6592   3rd Qu.:0.7000  
##                     Max.   :0.9333   Max.   :0.8750   Max.   :0.9333  
##                                                                       
##       ghc              bs1             bs2             bs3           bs4     
##  Min.   :0.5081   Min.   :0.886   Min.   :0.886   Min.   : NA   Min.   : NA  
##  1st Qu.:0.6029   1st Qu.:1.767   1st Qu.:1.767   1st Qu.: NA   1st Qu.: NA  
##  Median :0.6761   Median :2.106   Median :2.106   Median : NA   Median : NA  
##  Mean   :0.7117   Mean   :2.243   Mean   :2.243   Mean   :NaN   Mean   :NaN  
##  3rd Qu.:0.8042   3rd Qu.:2.472   3rd Qu.:2.472   3rd Qu.: NA   3rd Qu.: NA  
##  Max.   :0.9664   Max.   :5.617   Max.   :5.617   Max.   : NA   Max.   : NA  
##                                                   NA's   :100   NA's   :100  
##       bs5           bs6           auc1             auc2             auc3       
##  Min.   : NA   Min.   : NA   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.: NA   1st Qu.: NA   1st Qu.:0.3854   1st Qu.:0.3972   1st Qu.:0.5000  
##  Median : NA   Median : NA   Median :0.5556   Median :0.5714   Median :0.6500  
##  Mean   :NaN   Mean   :NaN   Mean   :0.5518   Mean   :0.5718   Mean   :0.6498  
##  3rd Qu.: NA   3rd Qu.: NA   3rd Qu.:0.7500   3rd Qu.:0.7500   3rd Qu.:0.8000  
##  Max.   : NA   Max.   : NA   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##  NA's   :100   NA's   :100                                                     
##       auc4             auc5             auc6             auc7       
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.5000   1st Qu.:0.5000   1st Qu.:0.5000   1st Qu.:0.5000  
##  Median :0.6500   Median :0.6500   Median :0.6500   Median :0.6500  
##  Mean   :0.6484   Mean   :0.6484   Mean   :0.6484   Mean   :0.6484  
##  3rd Qu.:0.8333   3rd Qu.:0.8333   3rd Qu.:0.8333   3rd Qu.:0.8333  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##                                                                     
##       auc8             auc9            auc10            auc11       
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.4625   1st Qu.:0.4500   1st Qu.:0.4500   1st Qu.:0.4486  
##  Median :0.6500   Median :0.6270   Median :0.6583   Median :0.6464  
##  Mean   :0.6333   Mean   :0.6212   Mean   :0.6233   Mean   :0.6143  
##  3rd Qu.:0.8333   3rd Qu.:0.8000   3rd Qu.:0.8031   3rd Qu.:0.8031  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##                                                                     
##      auc12            auc13            auc14            auc15       
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.4332   1st Qu.:0.4332   1st Qu.:0.4332   1st Qu.:0.3333  
##  Median :0.6460   Median :0.6460   Median :0.6460   Median :0.5298  
##  Mean   :0.6112   Mean   :0.6112   Mean   :0.6112   Mean   :0.5364  
##  3rd Qu.:0.8177   3rd Qu.:0.8177   3rd Qu.:0.8177   3rd Qu.:0.7500  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##                                                                     
##       auc        
##  Min.   :0.1429  
##  1st Qu.:0.4593  
##  Median :0.6024  
##  Mean   :0.5992  
##  3rd Qu.:0.7538  
##  Max.   :0.9597  
## 

characteristic_table(dnnsurv1)[[1]]

## [1] 0.8

1.9 gse1

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=empty_list
cox2=empty_list
cox3=empty_list
cox4=empty_list
pcox1=readRDS("savedresults/gse1_p_cox1.rds")
pcox2=readRDS("savedresults/gse1_p_cox2.rds")
pcox3=readRDS("savedresults/gse1_p_cox3.rds")
rsf1=readRDS("savedresults/gse1_rsf1.rds")
rsf2=readRDS("savedresults/gse1_rsf2.rds")
mtlr1=readRDS("savedresults/gse1_mtlr.rds")
dnnsurv1=empty_list
coxboost=readRDS("savedresults/gse1_coxboost.rds")

gacox=readRDS("savedresults/gse1_ga_cox1.rds")
gamtlr=readRDS("savedresults/gse1_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/gse1_ga_coxboost.rds")
limmamtlr=readRDS("savedresults/gse1_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/gse1_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/gse1_survivalsvm.rds")
deepsurv=readRDS("savedresults/gse1_deepsurv.rds")
deephit=readRDS("savedresults/gse1_deephit.rds")

want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

gse1full=want[[1]]
g64=want[[2]]
g65=want[[3]]
g66=want[[4]]
g67=want[[5]]
g68=want[[6]]
g69=want[[7]]
g70=want[[8]]

characteristic_table(pcox1)[[2]]

##     model                 hc               bc              unoc       
##  Length:100         Min.   :0.2828   Min.   :0.4326   Min.   :0.0000  
##  Class :character   1st Qu.:0.4914   1st Qu.:0.4960   1st Qu.:0.2035  
##  Mode  :character   Median :0.5216   Median :0.5217   Median :0.4891  
##                     Mean   :0.5263   Mean   :0.5850   Mean   :0.4593  
##                     3rd Qu.:0.5798   3rd Qu.:0.5526   3rd Qu.:0.7034  
##                     Max.   :0.6675   Max.   :1.0000   Max.   :0.9995  
##                                                                       
##       ghc              bs1               bs2               bs3     
##  Min.   :0.0000   Min.   :   36.3   Min.   :   36.3   Min.   : NA  
##  1st Qu.:0.5193   1st Qu.:  166.6   1st Qu.:  166.6   1st Qu.: NA  
##  Median :0.5555   Median :  344.9   Median :  344.9   Median : NA  
##  Mean   :0.4960   Mean   :  830.8   Mean   :  830.8   Mean   :NaN  
##  3rd Qu.:0.6060   3rd Qu.:  930.9   3rd Qu.:  930.9   3rd Qu.: NA  
##  Max.   :0.8125   Max.   :13466.5   Max.   :13466.5   Max.   : NA  
##                                                       NA's   :100  
##       bs4           bs5           bs6           auc1             auc2       
##  Min.   : NA   Min.   : NA   Min.   : NA   Min.   :0.2339   Min.   :0.2230  
##  1st Qu.: NA   1st Qu.: NA   1st Qu.: NA   1st Qu.:0.5000   1st Qu.:0.4991  
##  Median : NA   Median : NA   Median : NA   Median :0.5397   Median :0.5346  
##  Mean   :NaN   Mean   :NaN   Mean   :NaN   Mean   :0.5494   Mean   :0.5422  
##  3rd Qu.: NA   3rd Qu.: NA   3rd Qu.: NA   3rd Qu.:0.6237   3rd Qu.:0.6223  
##  Max.   : NA   Max.   : NA   Max.   : NA   Max.   :0.8539   Max.   :0.8386  
##  NA's   :100   NA's   :100   NA's   :100                                    
##       auc3             auc4             auc5             auc6       
##  Min.   :0.2192   Min.   :0.2192   Min.   :0.2192   Min.   :0.1202  
##  1st Qu.:0.4759   1st Qu.:0.4839   1st Qu.:0.4840   1st Qu.:0.4547  
##  Median :0.5028   Median :0.5089   Median :0.5259   Median :0.5000  
##  Mean   :0.5266   Mean   :0.5410   Mean   :0.5501   Mean   :0.5107  
##  3rd Qu.:0.6022   3rd Qu.:0.6279   3rd Qu.:0.6479   3rd Qu.:0.5779  
##  Max.   :0.7431   Max.   :0.7726   Max.   :0.8464   Max.   :0.7812  
##                                                                     
##       auc7             auc8              auc9            auc10       
##  Min.   :0.1179   Min.   :0.08111   Min.   :0.1401   Min.   :0.1401  
##  1st Qu.:0.4662   1st Qu.:0.44623   1st Qu.:0.4156   1st Qu.:0.4142  
##  Median :0.5000   Median :0.50000   Median :0.5000   Median :0.5000  
##  Mean   :0.5298   Mean   :0.51724   Mean   :0.4984   Mean   :0.4956  
##  3rd Qu.:0.6141   3rd Qu.:0.60407   3rd Qu.:0.5886   3rd Qu.:0.5891  
##  Max.   :0.8143   Max.   :0.80795   Max.   :0.8070   Max.   :0.9260  
##                                                                      
##      auc11             auc12             auc13             auc14        
##  Min.   :0.06721   Min.   :0.07735   Min.   :0.04917   Min.   :0.01518  
##  1st Qu.:0.36094   1st Qu.:0.39947   1st Qu.:0.36693   1st Qu.:0.36549  
##  Median :0.48841   Median :0.50000   Median :0.50000   Median :0.50000  
##  Mean   :0.46363   Mean   :0.48358   Mean   :0.45869   Mean   :0.47460  
##  3rd Qu.:0.51891   3rd Qu.:0.55220   3rd Qu.:0.54834   3rd Qu.:0.55976  
##  Max.   :0.92601   Max.   :0.93106   Max.   :0.93973   Max.   :0.93220  
##                                                                         
##      auc15              auc        
##  Min.   :0.05104   Min.   :0.2182  
##  1st Qu.:0.47657   1st Qu.:0.4880  
##  Median :0.50839   Median :0.5311  
##  Mean   :0.55389   Mean   :0.5264  
##  3rd Qu.:0.69182   3rd Qu.:0.5871  
##  Max.   :0.90737   Max.   :0.7237  
## 

1.10 gse2

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=empty_list
cox2=empty_list
cox3=empty_list
cox4=empty_list
pcox1=readRDS("savedresults/gse4_p_cox2.rds")
pcox2=readRDS("savedresults/gse4_p_cox2.rds")
pcox3=readRDS("savedresults/gse4_p_cox3.rds")
rsf1=readRDS("savedresults/gse4_rsf1.rds") 
rsf2=readRDS("savedresults/gse4_rsf1.rds") 
#mtlr1=readRDS("savedresults/gse4_mtlr.rds")
mtlr1=empty_list
dnnsurv1=empty_list
coxboost=readRDS("savedresults/gse4_coxboost.rds") 

gacox=readRDS("savedresults/gse4_ga_cox1.rds")
gamtlr=readRDS("savedresults/gse4_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/gse4_ga_coxboost.rds")
limmamtlr=readRDS("savedresults/gse4_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/gse4_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/gse4_survivalsvm.rds")
deepsurv=empty_list
deephit=empty_list
want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

gse2full=want[[1]]
g71=want[[2]]
g72=want[[3]]
g73=want[[4]]
g74=want[[5]]
g75=want[[6]]
g76=want[[7]]
g77=want[[8]]

characteristic_table(rsf1)[[2]]

##     model                 hc               bc           unoc       
##  Length:100         Min.   :0.1471   Min.   : NA   Min.   :0.0000  
##  Class :character   1st Qu.:0.4808   1st Qu.: NA   1st Qu.:0.0000  
##  Mode  :character   Median :0.5000   Median : NA   Median :0.0000  
##                     Mean   :0.4739   Mean   :NaN   Mean   :0.1718  
##                     3rd Qu.:0.5000   3rd Qu.: NA   3rd Qu.:0.3775  
##                     Max.   :0.7000   Max.   : NA   Max.   :0.7092  
##                                      NA's   :100                   
##       ghc           bs1             bs2             bs3           bs4     
##  Min.   : NA   Min.   :1.333   Min.   :1.333   Min.   : NA   Min.   : NA  
##  1st Qu.: NA   1st Qu.:2.108   1st Qu.:2.108   1st Qu.: NA   1st Qu.: NA  
##  Median : NA   Median :2.569   Median :2.569   Median : NA   Median : NA  
##  Mean   :NaN   Mean   :2.648   Mean   :2.648   Mean   :NaN   Mean   :NaN  
##  3rd Qu.: NA   3rd Qu.:3.022   3rd Qu.:3.022   3rd Qu.: NA   3rd Qu.: NA  
##  Max.   : NA   Max.   :4.360   Max.   :4.360   Max.   : NA   Max.   : NA  
##  NA's   :100                                   NA's   :100   NA's   :100  
##       bs5              bs6              auc1             auc2       
##  Min.   :0.1619   Min.   :0.1619   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.1929   1st Qu.:0.1929   1st Qu.:0.4501   1st Qu.:0.5000  
##  Median :0.2138   Median :0.2138   Median :0.5000   Median :0.5000  
##  Mean   :0.2188   Mean   :0.2188   Mean   :0.4559   Mean   :0.4641  
##  3rd Qu.:0.2338   3rd Qu.:0.2338   3rd Qu.:0.5000   3rd Qu.:0.5000  
##  Max.   :0.3140   Max.   :0.3140   Max.   :0.7500   Max.   :0.8125  
##                                                                     
##       auc3              auc4              auc5             auc6       
##  Min.   :0.05556   Min.   :0.05556   Min.   :0.0625   Min.   :0.0000  
##  1st Qu.:0.50000   1st Qu.:0.50000   1st Qu.:0.4910   1st Qu.:0.4600  
##  Median :0.50000   Median :0.50000   Median :0.5000   Median :0.5000  
##  Mean   :0.48997   Mean   :0.48114   Mean   :0.4593   Mean   :0.4497  
##  3rd Qu.:0.50000   3rd Qu.:0.50000   3rd Qu.:0.5000   3rd Qu.:0.5000  
##  Max.   :0.81250   Max.   :0.81250   Max.   :0.8333   Max.   :0.8333  
##                                                                       
##       auc7             auc8             auc9            auc10       
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.4305   1st Qu.:0.4305   1st Qu.:0.4305   1st Qu.:0.4305  
##  Median :0.5000   Median :0.5000   Median :0.5000   Median :0.5000  
##  Mean   :0.4445   Mean   :0.4445   Mean   :0.4445   Mean   :0.4465  
##  3rd Qu.:0.5000   3rd Qu.:0.5000   3rd Qu.:0.5000   3rd Qu.:0.5000  
##  Max.   :0.6764   Max.   :0.6764   Max.   :0.6764   Max.   :0.7500  
##                                                                     
##      auc11            auc12            auc13            auc14       
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000   Min.   :0.0000  
##  1st Qu.:0.4305   1st Qu.:0.4936   1st Qu.:0.4340   1st Qu.:0.4323  
##  Median :0.5000   Median :0.5000   Median :0.5000   Median :0.5000  
##  Mean   :0.4512   Mean   :0.4667   Mean   :0.4518   Mean   :0.4573  
##  3rd Qu.:0.5000   3rd Qu.:0.5000   3rd Qu.:0.5000   3rd Qu.:0.5000  
##  Max.   :1.0000   Max.   :1.0000   Max.   :1.0000   Max.   :1.0000  
##                                                                     
##      auc15             auc        
##  Min.   :0.0000   Min.   :0.0795  
##  1st Qu.:0.3932   1st Qu.:0.4435  
##  Median :0.5000   Median :0.5000  
##  Mean   :0.4438   Mean   :0.4588  
##  3rd Qu.:0.5000   3rd Qu.:0.5000  
##  Max.   :1.0000   Max.   :0.6883  
## 

1.11 ngene1

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=empty_list
cox2=empty_list
cox3=empty_list
cox4=empty_list
pcox1=readRDS("savedresults/ngse1_p_cox1.rds")
pcox2=readRDS("savedresults/ngse1_p_cox2.rds")
pcox3=readRDS("savedresults/ngse1_p_cox3.rds")
rsf1=readRDS("savedresults/ngse1_rsf1.rds") 
rsf2=readRDS("savedresults/ngse1_rsf1.rds") 
mtlr1=readRDS("savedresults/ngse1_mtlr.rds") 
dnnsurv1=readRDS("savedresults/ngse1_dnnsurv.rds") 
coxboost=readRDS("savedresults/ngse1_coxboost.rds") 

gacox=readRDS("savedresults/ngse1_ga_cox1.rds")
gamtlr=readRDS("savedresults/ngse1_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/ngse1_ga_cocboost.rds")
limmamtlr=readRDS("savedresults/ngse1_limma_mtlr.rds")
#limmacoxboost=readRDS("NGSE/ngse1_limma_coxboost.rds")
limmacoxboost=empty_list
survivalsvm=readRDS("savedresults/ngse1_survivalsvm.rds")
deepsurv=empty_list
deephit=empty_list

want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

ngse1full=want[[1]]
g85=want[[2]]
g86=want[[3]]
g87=want[[4]]
g88=want[[5]]
g89=want[[6]]
g90=want[[7]]
g91=want[[8]]

characteristic_table(dnnsurv1)[[1]]

## [1] 0.95

1.12 ngene2

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=empty_list
cox2=empty_list
cox3=empty_list
cox4=empty_list
pcox1=readRDS("savedresults/ngse2_p_cox1.rds")
pcox2=readRDS("savedresults/ngse2_p_cox2.rds")
pcox3=readRDS("savedresults/ngse2_p_cox3.rds")
rsf1=readRDS("savedresults/ngse2_rsf1.rds") 
rsf2=readRDS("savedresults/ngse2_rsf1.rds") 
mtlr1=readRDS("savedresults/ngse2_mtlr.rds") 
dnnsurv1=readRDS("savedresults/ngse2_dnnsurv.rds") 
coxboost=readRDS("savedresults/ngse2_coxboost.rds") 

gacox=readRDS("savedresults/ngse2_ga_cox1.rds")
gamtlr=readRDS("savedresults/ngse2_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/ngse2_ga_cocboost.rds")
limmamtlr=readRDS("savedresults/ngse2_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/ngse2_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/ngse2_survivalsvm.rds")
deepsurv=readRDS("savedresults/ngse2_deepsurv.rds")
deephit=readRDS("savedresults/ngse2_deephit.rds")

want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

ngse2full=want[[1]]
g92=want[[2]]
g93=want[[3]]
g94=want[[4]]
g95=want[[5]]
g96=want[[6]]
g97=want[[7]]
g98=want[[8]]
characteristic_table(dnnsurv1)[[1]]

## [1] 0.98

#ggsave(plot=g97,file="ge2_auc.pdf",device = "pdf")

1.13 ngene3

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=empty_list
cox2=empty_list
cox3=empty_list
cox4=empty_list
pcox1=readRDS("savedresults/ngse3_p_cox1.rds")
pcox2=readRDS("savedresults/ngse3_p_cox2.rds")
pcox3=readRDS("savedresults/ngse3_p_cox3.rds")
rsf1=readRDS("savedresults/ngse3_rsf1.rds") 
rsf2=readRDS("savedresults/ngse3_rsf1.rds") 
mtlr1=readRDS("savedresults/ngse3_mtlr.rds") 
dnnsurv1=readRDS("savedresults/ngse3_dnnsurv.rds") 
coxboost=readRDS("savedresults/ngse3_coxboost.rds") 

gacox=readRDS("savedresults/ngse3_ga_cox1.rds")
gamtlr=readRDS("savedresults/ngse3_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/ngse3_ga_cocboost.rds")
limmamtlr=readRDS("savedresults/ngse3_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/ngse3_limma_coxboost.rds")
survivalsvm=empty_list

deepsurv=readRDS("savedresults/ngse3_deepsurv.rds")
deephit=readRDS("savedresults/ngse3_deephit.rds")

want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

ngse3full=want[[1]]
g99=want[[2]]
g100=want[[3]]
g101=want[[4]]
g102=want[[5]]
g103=want[[6]]
g104=want[[7]]
g105=want[[8]]

characteristic_table(dnnsurv1)[[1]]

## [1] 0.97

1.14 ngene4

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=empty_list
cox2=empty_list
cox3=empty_list
cox4=empty_list
pcox1=readRDS("savedresults/ngse4_p_cox1.rds")
pcox2=readRDS("savedresults/ngse4_p_cox2.rds")
pcox3=readRDS("savedresults/ngse4_p_cox3.rds")
rsf1=readRDS("savedresults/ngse4_rsf1.rds") 
rsf2=readRDS("savedresults/ngse4_rsf1.rds") 
mtlr1=readRDS("savedresults/ngse4_mtlr.rds") 
dnnsurv1=readRDS("savedresults/ngse4_dnnsurv.rds") 
coxboost=readRDS("savedresults/ngse4_coxboost.rds") 

gacox=readRDS("savedresults/ngse4_ga_cox1.rds")
gamtlr=readRDS("savedresults/ngse4_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/ngse4_ga_cocboost.rds")
limmamtlr=readRDS("savedresults/ngse4_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/ngse4_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/ngse4_survivalsvm.rds")

deepsurv=readRDS("savedresults/ngse4_deepsurv.rds")
deephit=readRDS("savedresults/ngse4_deephit.rds")

want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

ngse4full=want[[1]]
g106=want[[2]]
g107=want[[3]]
g108=want[[4]]
g109=want[[5]]
g110=want[[6]]
g111=want[[7]]
g112=want[[8]]

characteristic_table(dnnsurv1)[[1]]

## [1] 1

#ggsave(plot=g111,file="ge4_auc.pdf",device = "pdf")

1.15 ngene5

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=empty_list
cox2=empty_list
cox3=empty_list
cox4=empty_list
pcox1=readRDS("savedresults/ngse5_p_cox1.rds")
pcox2=readRDS("savedresults/ngse5_p_cox2.rds")
pcox3=readRDS("savedresults/ngse5_p_cox3.rds")
rsf1=readRDS("savedresults/ngse5_rsf1.rds") 
rsf2=readRDS("savedresults/ngse5_rsf1.rds") 
mtlr1=readRDS("savedresults/ngse5_mtlr.rds") 
dnnsurv1=readRDS("savedresults/ngse5_dnnsurv.rds") 
coxboost=readRDS("savedresults/ngse5_coxboost.rds") 

gacox=readRDS("savedresults/ngse5_ga_cox1.rds")
gamtlr=readRDS("savedresults/ngse5_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/ngse5_ga_cocboost.rds")
limmamtlr=readRDS("savedresults/ngse5_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/ngse5_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/ngse5_survivalsvm.rds")

deepsurv=readRDS("savedresults/ngse5_deepsurv.rds")
deephit=readRDS("savedresults/ngse5_deephit.rds")

want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

ngse5full=want[[1]]
g113=want[[2]]
g114=want[[3]]
g115=want[[4]]
g116=want[[5]]
g117=want[[6]]
g118=want[[7]]
g119=want[[8]]
characteristic_table(dnnsurv1)[[1]]

## [1] 1

1.16 ngene6

empty_list=list()
for(i in 1:50){
  empty_list[[i]]=data.frame(matrix(NA, ncol = 26, nrow = 5))
  colnames(empty_list[[i]])=c("hc_acc5", "bc_acc5", "unoc_acc5" ,"ghc_acc5" , "bs1" , "bs2" ,"bs3","bs4",  "bs5", "bs6" , "auc1"  , "auc2","auc3" , "auc4","auc5" , "auc6", "auc7", "auc8", "auc9", "auc10" , "auc11" , "auc12" , "auc13",  "auc14" , "auc15" , "auc")
}
cox1=empty_list
cox2=empty_list
cox3=empty_list
cox4=empty_list
pcox1=readRDS("savedresults/ngse6_p_cox1.rds")
pcox2=readRDS("savedresults/ngse6_p_cox2.rds")
pcox3=readRDS("savedresults/ngse6_p_cox3.rds")
rsf1=readRDS("savedresults/ngse6_rsf1.rds") 
rsf2=readRDS("savedresults/ngse6_rsf1.rds") 
mtlr1=readRDS("savedresults/ngse6_mtlr.rds") 
dnnsurv1=readRDS("savedresults/ngse6_dnnsurv.rds") 
coxboost=readRDS("savedresults/ngse6_coxboost.rds") 

gacox=readRDS("savedresults/ngse6_ga_cox1.rds")
gamtlr=readRDS("savedresults/ngse6_ga_mtlr.rds")
gacoxboost=readRDS("savedresults/ngse6_ga_cocboost.rds")
limmamtlr=readRDS("savedresults/ngse6_limma_mtlr.rds")
limmacoxboost=readRDS("savedresults/ngse6_limma_coxboost.rds")
survivalsvm=readRDS("savedresults/ngse6_survivalsvm.rds")

deepsurv=readRDS("savedresults/ngse6_deepsurv.rds")
deephit=readRDS("savedresults/ngse6_deephit.rds")

want=plot_fun(cox1, cox2, cox3, cox4, pcox1, pcox2, pcox3, rsf1, rsf2, mtlr1, dnnsurv1,coxboost,gacox,gamtlr,gacoxboost,limmamtlr,limmacoxboost,survivalsvm,deepsurv,deephit)

ngse6full=want[[1]]
g120=want[[2]]
g121=want[[3]]
g122=want[[4]]
g123=want[[5]]
g124=want[[6]]
g125=want[[7]]
g126=want[[8]]

characteristic_table(dnnsurv1)[[1]]

## [1] 1

2 each plot

labelss<-c("pbc","veteran","lung","anz","us","melanoma_clinical","melanoma_itraq","melanoma_nano","gse1","gse2","gene1","gene2","gene3","gene4","gene5","gene6")
ggarrange(g1,g8,g15,g22,g29,g36,g43,g57,g64,g71,g85,g92,g99,g106,g113,g120,labels = labelss,ncol = 3, nrow = 6) #miss g78 for gse3

ggarrange(g6,g13, g20,g27,g34,g41,g48,g62,g69,g76,g90,g97,g104,g111,g118,g125,labels = labelss,ncol = 3, nrow = 6) #g83

ggarrange(g5,g12, g19,g26,g33,g40,g47,g61,g68,g75,g89,g96,g103,g110,g117,g124,labels = labelss,ncol = 3, nrow = 6) #g82

# ggarrange(g2,g9, g16,g23,g30,g35,g44,g51,g58,g65,g72,g79,labels = labelss,ncol = 3, nrow = 4)
# ggarrange(g4,g11, g18,g25,g32,g37,g46,g53,g60,g67,g74,g81,labels = labelss,ncol = 3, nrow = 4)
#g97 #gene2_auc.pdf

3 combined plot

3.1 data summary

cen=read_excel("result.xlsx",sheet = 2)
#kable(cen)
library(kableExtra)
kable(cen, "html") %>%
  kable_styling("striped") %>%
  save_kable("table.png")

#save_kable(kable(cen),file="table.png")

3.2 feasibility summary

plotdt=read_excel("result.xlsx",sheet = 1)
plotdt=as.data.frame(plotdt)
rownames(plotdt)=plotdt$method
#pheatmap(as.matrix(plotdt[,-1]),cluster_rows = FALSE,cluster_cols = TRUE)
#col_fun = colorRamp2(c( 0, 1), c("seagreen", "lightpink"))
nb.cols <- 18
mycolors <- colorRampPalette(brewer.pal(8, "RdBu"))(nb.cols)
mat=as.matrix(plotdt[,-1])
Heatmap(mat, name = "index", col = mycolors,cluster_rows = FALSE,rect_gp = gpar(col = "white", lwd = 2))

#manually save heatmap here


indicator=read_excel("result.xlsx",sheet = 11)
pheatmap(indicator[,-1],cluster_cols = FALSE, cluster_rows = FALSE,color = c("orange","purple","pink"))

plotdt=read_excel("result.xlsx",sheet = 7)
plotdt=as.data.frame(plotdt)
rownames(plotdt)=plotdt$method
#pheatmap(as.matrix(plotdt[,-1]),cluster_rows = FALSE,cluster_cols = TRUE)
# col_fun = colorRamp2(c( 0, 1), c("blue", "red"))
mat=as.matrix(plotdt[,-1])
Heatmap(mat, name = "index", col = mycolors,cluster_rows = FALSE,rect_gp = gpar(col = "white", lwd = 2))

3.3 clinical vs omics

data_list=list(anzfull,usfull,veteranfull,lungfull,pbcfull,melanoma_clinicalfull,melanoma_itraqfull,melanomananofull,gse1full,gse2full,ngse1full,ngse2full,ngse3full,ngse4full,ngse5full,ngse6full)
datafull=bind_rows(data_list, .id = "datasets")

nb.cols <- 20
mycolors <- colorRampPalette(brewer.pal(8, "RdBu"))(nb.cols)

plotdt=datafull[datafull$model=="rsf2",]
plotdt=plotdt[plotdt$variable=="bs5",]
plotdt2=plotdt[!is.infinite(plotdt$value),]
#plotdt2$observations=as.numeric(mapvalues(plotdt2$datasets, from = seq(1,11,1), to = c(3323,3000,137,228,312,88,41,70,45,194,58)))
#plotdt2=plotdt2 %>% arrange(observations)
#plotdt$datasets=factor(plotdt$datasets,levels=c("anzfull","usfull","veteranfull","lungfull","ovarianfull","melanoma_clinicalfull","melanoma_itraqfull","melanoma_swathfull","ovarian1full","ovarian2full"))
plotdt2$datasets=mapvalues(plotdt2$datasets, from = seq(1,16,1), to = c("anz3323","us3000","veteran137","lung228","pbc312","melanoma_clinical88","melanoma_itraq41","melanoma_nano45","gse1 194","gse2 58","ngene1 115","ngene2 295","mgene3 86","ngene4 116","ngene5 78","ngene6 240"))
plotdt2$datasets=factor(plotdt2$datasets,levels=c("melanoma_itraq41","melanoma_nano45","gse2 58","ngene5 78","mgene3 86","ngene1 115","ngene4 116","gse1 194","ngene6 240","ngene2 295","melanoma_clinical88","veteran137","lung228","pbc312","us3000","anz3323"))
p=ggplot(plotdt2, aes(x=datasets, y=value, fill=datasets)) + geom_boxplot()+scale_y_continuous(limits=c(0,0.25))+ scale_fill_manual(values = mycolors)+theme_bw()
#ggsave(plot = p,file="figures/figure3c1.pdf",device = "pdf")


plotdt=datafull[datafull$model=="coxboost",]
plotdt=plotdt[plotdt$variable=="hc",]
plotdt2=plotdt[!is.infinite(plotdt$value),]

#plotdt$datasets=factor(plotdt$datasets,levels=c("anzfull","usfull","veteranfull","lungfull","ovarianfull","melanoma_clinicalfull","melanoma_itraqfull","melanoma_swathfull","ovarian1full","ovarian2full"))
plotdt2$datasets=mapvalues(plotdt2$datasets, from = seq(1,16,1), to = c("anz3323","us3000","veteran137","lung228","pbc312","melanoma_clinical88","melanoma_itraq41","melanoma_nano45","gse1 194","gse2 58","ngene1 115","ngene2 295","mgene3 86","ngene4 116","ngene5 78","ngene6 240"))
plotdt2$datasets=factor(plotdt2$datasets,levels=c("melanoma_itraq41","melanoma_nano45","gse2 58","ngene5 78","mgene3 86","ngene1 115","ngene4 116","gse1 194","ngene6 240","ngene2 295","melanoma_clinical88","veteran137","lung228","pbc312","us3000","anz3323"))
p=ggplot(plotdt2, aes(x=datasets, y=value, fill=datasets)) + geom_boxplot()+scale_y_continuous(limits=c(0,1))+ scale_fill_manual(values = mycolors)+theme_bw()
p

#ggsave(plot = p,file="figures/figure3c2.pdf",device = "pdf")

3.4 mean hc (data*method)

plotdt=datafull[datafull$variable=="hc",]
plotdt2 <- plotdt %>%dplyr::group_by(datasets,model) %>%dplyr::summarize(Mean = mean(value, na.rm=TRUE))
plotdt2$datasets=plyr::mapvalues(plotdt2$datasets, from = seq(1,16,1), to = c("anz","us","veteran","lung","pbc","melanoma_clinical","melanoma_itraq","melanoma_nano","gse1","gse2","ngene1","ngene2","mgene3","ngene4","ngene5","ngene6"))
plotdt2$datasets=factor(plotdt2$datasets,levels=c("anz","us","veteran","lung","pbc","melanoma_clinical","melanoma_itraq","melanoma_nano","gse1","gse2","ngene1","ngene2","mgene3","ngene4","ngene5","ngene6"))
plotdt2$model=factor(plotdt2$model,levels=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","coxboost","gacox","gacoxboost","limmacoxboost","mtlr1","gamtlr","limmamtlr","rsf1","rsf2","survivalsvm","deephit","deepsurv","dnnsurv1"))
mat=tidyr::spread(plotdt2, model, Mean)
mat=as.data.frame(mat)
rownames(mat)=mat$datasets
mat=as.matrix(mat[,-1])
nb.cols <- 8
mycolors <- colorRampPalette(rev(brewer.pal(8, "RdBu")))(nb.cols)
Heatmap(t(mat), name = "index", cluster_rows = FALSE,cluster_columns = FALSE,rect_gp = gpar(col = "white", lwd = 2),col = mycolors)

3.5 some picked comparison plots

#1:anz
#16:ngene5
plotdt=datafull[datafull$model%in% c("pcox3","pcox1","coxboost","gacoxboost"),]
plotdt=plotdt[plotdt$variable=="hc"& plotdt$datasets==1,]
plotdt2=plotdt[!is.infinite(plotdt$value),]
plotdt2$model=factor(plotdt2$model,levels = c("pcox1","pcox3","coxboost","gacoxboost"))
ggplot(plotdt2, aes(x=model, y=value, fill=model)) + geom_boxplot()+theme_bw()

#ggsave(plot=ggplot(plotdt2, aes(x=model, y=value, fill=model)) + geom_boxplot()+theme_bw(),file="figure2a1.pdf",device = "pdf")

plotdt=datafull[datafull$model%in% c("pcox3","pcox1","coxboost","gacoxboost"),]
plotdt=plotdt[plotdt$variable=="bs1"& plotdt$datasets==1,]
plotdt2=plotdt[!is.infinite(plotdt$value),]
plotdt2$model=factor(plotdt2$model,levels = c("pcox1","pcox3","coxboost","gacoxboost"))
ggplot(plotdt2, aes(x=model, y=value, fill=model)) + geom_boxplot()+theme_bw()

#ggsave(plot=ggplot(plotdt2, aes(x=model, y=value, fill=model)) + geom_boxplot()+theme_bw(),file="figure2a2.pdf",device = "pdf")

plotdt=datafull[datafull$model%in% c("pcox3","pcox1","coxboost","gacoxboost"),]
plotdt=plotdt[plotdt$variable=="hc"& plotdt$datasets==16,]
plotdt2=plotdt[!is.infinite(plotdt$value),]
plotdt2$model=factor(plotdt2$model,levels = c("pcox1","pcox3","coxboost","gacoxboost"))
ggplot(plotdt2, aes(x=model, y=value, fill=model)) + geom_boxplot()+theme_bw()

#ggsave(ggplot(plotdt2, aes(x=model, y=value, fill=model)) + geom_boxplot()+theme_bw(),file="figure2a3.pdf",device = "pdf")

plotdt=datafull[datafull$model%in% c("pcox3","pcox1","coxboost","gacoxboost"),]
plotdt=plotdt[plotdt$variable=="bs1"& plotdt$datasets==16,]
plotdt2=plotdt[!is.infinite(plotdt$value),]
plotdt2$model=factor(plotdt2$model,levels = c("pcox1","pcox3","coxboost","gacoxboost"))
ggplot(plotdt2, aes(x=model, y=value, fill=model)) + geom_boxplot()+theme_bw()

#ggsave(ggplot(plotdt2, aes(x=model, y=value, fill=model)) + geom_boxplot()+theme_bw(),file="figure2a4.pdf",device = "pdf")


# save for figure3a

3.6 reproducibility

plottdt=read_excel("result.xlsx",sheet = 8)
plottdt=plottdt[11,]
plottdt=as.data.frame(t(plottdt))
plottdt$data=rownames(plottdt)
plottdt=plottdt[which(plottdt$V1>0),]
plottdt=plottdt[-1,]
plottdt$V1=as.numeric(plottdt$V1)
#aes(x = "", y = V1, fill = data )
#ggsave(ggplot(data = plottdt,aes(x = reorder(data, V1), y = V1,fill = data )) + geom_bar(stat = "identity", position = position_dodge())+theme_bw(),file="figure1d2.pdf",device = "pdf")

plottdt=data.frame(method=c("bw_coxaic","bw_coxp","bw_coxbic"),value=c(0.75,0.9,0.55))
#ggsave(ggplot(data = plottdt, aes(x = "", y = value, fill = method )) + geom_bar(stat = "identity", position =  position_dodge())+theme_bw(),file="figure1d1.pdf",device = "pdf")

4 supplymentary

4.1 linear model on cindex

check linear model: which aspect affects cindex

regressiondt=datafull%>% filter(variable=="hc")%>% group_by(datasets,model)%>% dplyr::summarise(medianc=median(value))
regressiondt$datasets=mapvalues(regressiondt$datasets, from = seq(1,11,1), to = c("anz","us","veteran","lung","pbc","melanoma_clinical","melanoma_itraq","melanoma_swath","melanoma_nano","gse1","gse2"))
tablee=read_excel("result.xlsx",sheet = 9)
regressiondt1 <- tablee %>% left_join(regressiondt, by = c("name" = "datasets"))
regressiondt2<-regressiondt1%>% distinct()

fitted=lm(medianc~.,data = regressiondt2[,-1])
summary(fitted)

## 
## Call:
## lm(formula = medianc ~ ., data = regressiondt2[, -1])
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.307263 -0.052409 -0.000292  0.043616  0.190176 
## 
## Coefficients:
##                      Estimate Std. Error t value Pr(>|t|)    
## (Intercept)         6.784e-01  4.134e-02  16.409  < 2e-16 ***
## num_obs            -2.374e-06  1.195e-05  -0.199 0.842798    
## num_feature         2.812e-06  1.358e-06   2.071 0.040095 *  
## censoring_rate     -7.277e-03  5.057e-02  -0.144 0.885769    
## variable_typenc    -5.051e-02  2.574e-02  -1.962 0.051595 .  
## data_typeomics     -1.149e-01  2.062e-02  -5.575 1.10e-07 ***
## modelcox2           5.610e-03  5.631e-02   0.100 0.920775    
## modelcox3           3.211e-02  5.998e-02   0.535 0.593169    
## modelcox4          -5.236e-02  5.631e-02  -0.930 0.353891    
## modelcoxboost      -1.255e-02  4.745e-02  -0.265 0.791700    
## modeldeephit       -1.024e-01  4.910e-02  -2.086 0.038605 *  
## modeldeepsurv      -9.319e-02  4.910e-02  -1.898 0.059569 .  
## modeldnnsurv1      -1.064e-02  5.198e-02  -0.205 0.838057    
## modelgacox         -1.694e-02  4.839e-02  -0.350 0.726789    
## modelgacoxboost    -1.353e-02  4.839e-02  -0.280 0.780124    
## modelgamtlr         1.766e-01  4.839e-02   3.650 0.000361 ***
## modellimmacoxboost -3.573e-02  4.817e-02  -0.742 0.459350    
## modellimmamtlr      1.918e-01  4.745e-02   4.043 8.38e-05 ***
## modelmtlr1          1.679e-01  4.903e-02   3.424 0.000794 ***
## modelpcox1         -5.101e-03  4.745e-02  -0.108 0.914529    
## modelpcox2         -1.543e-02  4.817e-02  -0.320 0.749122    
## modelpcox3          8.742e-03  4.817e-02   0.181 0.856240    
## modelrsf1           1.937e-02  4.745e-02   0.408 0.683601    
## modelrsf2           1.611e-02  4.745e-02   0.339 0.734710    
## modelsurvivalsvm   -5.044e-02  5.068e-02  -0.995 0.321216    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.09254 on 152 degrees of freedom
## Multiple R-squared:  0.5352, Adjusted R-squared:  0.4618 
## F-statistic: 7.293 on 24 and 152 DF,  p-value: 1.694e-15

head(regressiondt2)

## # A tibble: 6 x 8
##   name  num_obs num_feature censoring_rate variable_type data_type model medianc
##   <chr>   <dbl>       <dbl>          <dbl> <chr>         <chr>     <chr>   <dbl>
## 1 anz      3323          40          0.874 nc            clinical  cox1    0.620
## 2 anz      3323          40          0.874 nc            clinical  cox2    0.612
## 3 anz      3323          40          0.874 nc            clinical  cox3    0.620
## 4 anz      3323          40          0.874 nc            clinical  cox4    0.5  
## 5 anz      3323          40          0.874 nc            clinical  coxb…   0.631
## 6 anz      3323          40          0.874 nc            clinical  deep…   0.555

stargazer(fitted, type = "text")

## 
## ===============================================
##                         Dependent variable:    
##                     ---------------------------
##                               medianc          
## -----------------------------------------------
## num_obs                      -0.00000          
##                              (0.00001)         
##                                                
## num_feature                  0.00000**         
##                              (0.00000)         
##                                                
## censoring_rate                -0.007           
##                               (0.051)          
##                                                
## variable_typenc               -0.051*          
##                               (0.026)          
##                                                
## data_typeomics               -0.115***         
##                               (0.021)          
##                                                
## modelcox2                      0.006           
##                               (0.056)          
##                                                
## modelcox3                      0.032           
##                               (0.060)          
##                                                
## modelcox4                     -0.052           
##                               (0.056)          
##                                                
## modelcoxboost                 -0.013           
##                               (0.047)          
##                                                
## modeldeephit                 -0.102**          
##                               (0.049)          
##                                                
## modeldeepsurv                 -0.093*          
##                               (0.049)          
##                                                
## modeldnnsurv1                 -0.011           
##                               (0.052)          
##                                                
## modelgacox                    -0.017           
##                               (0.048)          
##                                                
## modelgacoxboost               -0.014           
##                               (0.048)          
##                                                
## modelgamtlr                  0.177***          
##                               (0.048)          
##                                                
## modellimmacoxboost            -0.036           
##                               (0.048)          
##                                                
## modellimmamtlr               0.192***          
##                               (0.047)          
##                                                
## modelmtlr1                   0.168***          
##                               (0.049)          
##                                                
## modelpcox1                    -0.005           
##                               (0.047)          
##                                                
## modelpcox2                    -0.015           
##                               (0.048)          
##                                                
## modelpcox3                     0.009           
##                               (0.048)          
##                                                
## modelrsf1                      0.019           
##                               (0.047)          
##                                                
## modelrsf2                      0.016           
##                               (0.047)          
##                                                
## modelsurvivalsvm              -0.050           
##                               (0.051)          
##                                                
## Constant                     0.678***          
##                               (0.041)          
##                                                
## -----------------------------------------------
## Observations                    177            
## R2                             0.535           
## Adjusted R2                    0.462           
## Residual Std. Error      0.093 (df = 152)      
## F Statistic           7.293*** (df = 24; 152)  
## ===============================================
## Note:               *p<0.1; **p<0.05; ***p<0.01

more added on 20210429, check linear model: which aspect affects cindex

regressiondt=datafull%>% filter(variable=="hc")%>% group_by(datasets,model)%>% dplyr::summarise(medianc=median(value))

regressiondt$datasets=mapvalues(regressiondt$datasets, from = seq(1,16,1), to = c("anz","us","veteran","lung","pbc","melanoma_clinical","melanoma_itraq","melanoma_nano","gse1","gse2","ngene1","ngene2","ngene3","ngene4","ngene5","ngene6"))

regressiondt=regressiondt[regressiondt$model=="rsf1",]
tablee=read_excel("result.xlsx",sheet = 10)
regressiondt1 <- tablee %>% left_join(regressiondt, by = c("Data" = "datasets"))
regressiondt2=regressiondt1[,c(3:9,13)]
fitted=lm(medianc~.,data = regressiondt2)
summary(fitted)
head(regressiondt2)
stargazer(fitted, type = "text")

heatmap of cindex for model vs data

olddata_long=regressiondt2[,c(1,7,8)]
data_wide <- tidyr::spread(olddata_long,model, medianc)
data_wide
plotdt=as.data.frame(data_wide)
rownames(plotdt)=plotdt$name
#pheatmap(as.matrix(plotdt[,-1]),cluster_rows = FALSE,cluster_cols = TRUE)
col_fun = colorRamp2(c( 0.4,1), c("seagreen", "lightpink"))
mat=as.matrix(plotdt[,-1])
mat[is.na(mat)] <- 0
Heatmap(mat, name = "index", col = col_fun,cluster_rows = FALSE,cluster_columns = FALSE,rect_gp = gpar(col = "white", lwd = 2))

check for this data pbc from another package: still, mtlr is bad

data("pbc",package = "survival")
dim(pbc)
colnames(pbc)

pbc1 <- pbc[,-1]
dim(pbc1)
any(is.na(pbc1))
pbc2=na.omit(pbc1)
dim(pbc2) #276 19
table(pbc2$status) 
pbc3=pbc2[pbc2$status!=1,]
pbc3$status=as.vector(ifelse(pbc3$status==2,1,0))
table(pbc3$status) #147 111
fitform_ogl=as.formula(Surv(time, status) ~ .)
summary(pbc3$time)

formula1=fitform_ogl
formula2=fitform_ogl
formula3=Surv(time,status)~1
formula4=Surv(time,status)~1
timess=seq(50,2000,(2000-50)/15)
#9

cox1 <- pbmcapply::pbmclapply(1:20, rsf2_fun,pbc3,5, fitform_ogl,formula1,formula2,formula3,formula4,timess, mc.cores = 2)

head(cox1)
colMeans(do.call(rbind, cox1))
#10

cox2 <- pbmcapply::pbmclapply(1:20, mtlr_fun,pbc3,5, fitform_ogl,formula1,formula2,formula3,formula4,timess, mc.cores = 2)

head(cox2)
colMeans(do.call(rbind, cox2))

4.2 time and memory full report

time_fun=function(model_list){
  time_vec=c()
for (i in 1:20){
  dt=model_list[[i]]
  if (is.null(dt)){time_vec[i]=NA} else {time_vec[i]=dt}
}
  return(time_vec)
}


memory_fun=function(model_list){
  memory_vec=c()
for (i in 1:20){
  dt=model_list[[i]]
  if (is.null(dt)){memory_vec[i]=NA} else {memory_vec[i]=sum(dt$by.total$mem.total)}
}
  return(memory_vec)
}

plot_fun1=function(valuee){
  df=data.frame(methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit"), value=valuee)
  p<-ggplot(data=df, aes(x=methods, y=value)) +
  geom_bar(stat="identity")+ggtitle("               memory(MB)")+theme_bw()
  return(p)
}

plot_fun2=function(valuee){
  df=data.frame(methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit"), value=valuee)
  p<-ggplot(data=df, aes(x=methods, y=value)) +
  geom_bar(stat="identity")+ggtitle("                time(seconds)")+theme_bw()
  return(p)
}

#pbc
cox1=readRDS("savedresults/pbc_cox1m.rds")
cox2=readRDS("savedresults/pbc_bw_cox1m.rds")
cox3=readRDS("savedresults/pbc_bw_cox2m.rds")
cox4=readRDS("savedresults/pbc_bw_cox3m.rds")
pcox1=readRDS("savedresults/pbc_p_cox1m.rds")
pcox2=readRDS("savedresults/pbc_p_cox2m.rds")
pcox3=readRDS("savedresults/pbc_p_cox3m.rds")
rsf1=readRDS("savedresults/pbc_rsf1m.rds")
rsf2=readRDS("savedresults/pbc_rsf2m.rds")
mtlr1=readRDS("savedresults/pbc_mtlrm.rds")
dnnsurv1=readRDS("savedresults/pbc_dnnsurvm.rds")
coxboost=readRDS("savedresults/pbc_coxboostm.rds")
survivalsvm=readRDS("savedresults/pbc_survivalsvmm.rds")
ga_cox=readRDS("savedresults/pbc_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/pbc_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/pbc_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/pbc_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/pbc_limma_coxboostm.rds")
deepsurv=readRDS("savedresults/pbc_deepsurvm.rds")
deephit=readRDS("savedresults/pbc_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)
tcox1=readRDS("savedresults/pbc_cox1t.rds")
tcox2=readRDS("savedresults/pbc_bw_cox1t.rds")
tcox3=readRDS("savedresults/pbc_bw_cox2t.rds")
tcox4=readRDS("savedresults/pbc_bw_cox3t.rds")
tpcox1=readRDS("savedresults/pbc_p_cox1t.rds")
tpcox2=readRDS("savedresults/pbc_p_cox2t.rds")
tpcox3=readRDS("savedresults/pbc_p_cox3t.rds")
trsf1=readRDS("savedresults/pbc_rsf1t.rds")
trsf2=readRDS("savedresults/pbc_rsf2t.rds")
tmtlr1=readRDS("savedresults/pbc_mtlrt.rds")
tdnnsurv1=readRDS("savedresults/pbc_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/pbc_coxboostt.rds")
tsurvivalsvm=readRDS("savedresults/pbc_survivalsvmt.rds")*60
tga_cox=readRDS("savedresults/pbc_ga_cox1t.rds")
tga_mtlr=readRDS("savedresults/pbc_ga_mtlrt.rds")
tga_coxboost=readRDS("savedresults/pbc_ga_coxboostt.rds")
tlimma_mtlr=readRDS("savedresults/pbc_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/pbc_limma_coxboostt.rds")
tdeepsurv=readRDS("savedresults/pbc_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/pbc_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)

m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g1=plot_fun1(m)
g2=plot_fun2(t)
pbcall=cbind.data.frame(m,t,m1,t1)
pbcall$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#veteran
cox1=readRDS("savedresults/veteran_cox1m.rds")
cox2=readRDS("savedresults/veteran_bw_cox1m.rds")
cox3=readRDS("savedresults/veteran_bw_cox2m.rds")
cox4=readRDS("savedresults/veteran_bw_cox3m.rds")
pcox1=readRDS("savedresults/veteran_p_cox1m.rds")
pcox2=readRDS("savedresults/veteran_p_cox2m.rds")
pcox3=readRDS("savedresults/veteran_p_cox3m.rds")
rsf1=readRDS("savedresults/veteran_rsf1m.rds")
rsf2=readRDS("savedresults/veteran_rsf2m.rds")
mtlr1=readRDS("savedresults/veteran_mtlrm.rds")
dnnsurv1=readRDS("savedresults/veteran_dnnsurvm.rds")
coxboost=readRDS("savedresults/veteran_coxboostm.rds")
survivalsvm=readRDS("savedresults/veteran_survivalsvmm.rds")
ga_cox=readRDS("savedresults/veteran_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/veteran_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/veteran_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/veteran_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/veteran_limma_coxboostm.rds")
deepsurv=readRDS("savedresults/pbc_deepsurvm.rds")
deephit=readRDS("savedresults/pbc_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)
tcox1=readRDS("savedresults/veteran_cox1t.rds")
tcox2=readRDS("savedresults/veteran_bw_cox1t.rds")
tcox3=readRDS("savedresults/veteran_bw_cox2t.rds")
tcox4=readRDS("savedresults/veteran_bw_cox3t.rds")
tpcox1=readRDS("savedresults/veteran_p_cox1t.rds")
tpcox2=readRDS("savedresults/veteran_p_cox2t.rds")
tpcox3=readRDS("savedresults/veteran_p_cox3t.rds")
trsf1=readRDS("savedresults/veteran_rsf1t.rds")
trsf2=readRDS("savedresults/veteran_rsf2t.rds")
tmtlr1=readRDS("savedresults/veteran_mtlrt.rds")
tdnnsurv1=readRDS("savedresults/veteran_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/veteran_coxboostt.rds")
tsurvivalsvm=readRDS("savedresults/veteran_survivalsvmt.rds")
tga_cox=readRDS("savedresults/veteran_ga_cox1t.rds")
tga_mtlr=readRDS("savedresults/veteran_ga_mtlrt.rds")
tga_coxboost=readRDS("savedresults/veteran_ga_coxboostt.rds")
tlimma_mtlr=readRDS("savedresults/veteran_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/veteran_limma_coxboostt.rds")
tdeepsurv=readRDS("savedresults/veteran_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/veteran_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g3=plot_fun1(m)
g4=plot_fun2(t)
veteranall=cbind.data.frame(m,t,m1,t1)
veteranall$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#lung
cox1=readRDS("savedresults/lung_cox1m.rds")
cox2=readRDS("savedresults/lung_bw_cox1m.rds")
cox3=readRDS("savedresults/lung_bw_cox2m.rds")
pcox1=readRDS("savedresults/lung_p_cox1m.rds")
pcox2=readRDS("savedresults/lung_p_cox2m.rds")
pcox3=readRDS("savedresults/lung_p_cox3m.rds")
rsf1=readRDS("savedresults/lung_rsf1m.rds")
rsf2=readRDS("savedresults/lung_rsf2m.rds")
mtlr1=readRDS("savedresults/lung_mtlrm.rds")
dnnsurv1=readRDS("savedresults/lung_dnnsurvm.rds")
coxboost=readRDS("savedresults/lung_coxboostm.rds")
cox4=readRDS("savedresults/lung_bw_cox3m.rds")
survivalsvm=readRDS("savedresults/lung_survivalsvmm.rds")
ga_cox=readRDS("savedresults/lung_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/lung_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/lung_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/lung_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/lung_limma_coxboostm.rds")
deepsurv=readRDS("savedresults/lung_deepsurvm.rds")
deephit=readRDS("savedresults/lung_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=readRDS("savedresults/lung_cox1t.rds")
tcox2=readRDS("savedresults/lung_bw_cox1t.rds")
tcox3=readRDS("savedresults/lung_bw_cox2t.rds")
tpcox1=readRDS("savedresults/lung_p_cox1t.rds")
tpcox2=readRDS("savedresults/lung_p_cox2t.rds")
tpcox3=readRDS("savedresults/lung_p_cox3t.rds")
trsf1=readRDS("savedresults/lung_rsf1t.rds")
trsf2=readRDS("savedresults/lung_rsf2t.rds")
tmtlr1=readRDS("savedresults/lung_mtlrt.rds")
tdnnsurv1=readRDS("savedresults/lung_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/lung_coxboostt.rds")
tcox4=readRDS("savedresults/lung_bw_cox3t.rds")
tsurvivalsvm=readRDS("savedresults/lung_survivalsvmt.rds")
tga_cox=readRDS("savedresults/lung_ga_cox1t.rds")
tga_mtlr=readRDS("savedresults/lung_ga_mtlrt.rds")
tga_coxboost=readRDS("savedresults/lung_ga_coxboostt.rds")
tlimma_mtlr=readRDS("savedresults/lung_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/lung_limma_coxboostt.rds")
tdeepsurv=readRDS("savedresults/lung_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/lung_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g5=plot_fun1(m)
g6=plot_fun2(t)
lungall=cbind.data.frame(m,t,m1,t1)
lungall$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")

#anz
cox1=readRDS("savedresults/anz_cox1m.rds")
cox2=readRDS("savedresults/anz_bw_cox1m.rds")
cox3=readRDS("savedresults/anz_bw_cox2m.rds")
cox4=readRDS("savedresults/anz_bw_cox3m.rds")
pcox1=readRDS("savedresults/anz_p_cox1m.rds")
pcox2=readRDS("savedresults/anz_p_cox2m.rds")
pcox3=readRDS("savedresults/anz_p_cox3m.rds")
rsf1=readRDS("savedresults/anz_rsf1m.rds")
rsf2=readRDS("savedresults/anz_rsf2m.rds")
mtlr1=readRDS("savedresults/anz_mtlrm.rds")
dnnsurv1=readRDS("savedresults/anz_dnnsurvm.rds")
coxboost=readRDS("savedresults/anz_coxboostm.rds")
survivalsvm=NULL
ga_cox=readRDS("savedresults/anz_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/anz_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/anz_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/anz_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/anz_limma_coxboostm.rds")
deepsurv=readRDS("savedresults/anz_deepsurvm.rds")
deephit=readRDS("savedresults/anz_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=readRDS("savedresults/anz_cox1t.rds")
tcox2=readRDS("savedresults/anz_bw_cox1t.rds")
tcox3=readRDS("savedresults/anz_bw_cox2t.rds")
tcox4=readRDS("savedresults/anz_bw_cox3t.rds")
tpcox1=readRDS("savedresults/anz_p_cox1t.rds")*60
tpcox2=readRDS("savedresults/anz_p_cox2t.rds")
tpcox3=readRDS("savedresults/anz_p_cox3t.rds")*60
trsf1=readRDS("savedresults/anz_rsf1t.rds")*60
trsf2=readRDS("savedresults/anz_rsf2t.rds")*60
tmtlr1=readRDS("savedresults/anz_mtlrt.rds")*60
tdnnsurv1=readRDS("savedresults/anz_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/anz_coxboostt.rds")
tsurvivalsvm=NULL
tga_cox=readRDS("savedresults/anz_ga_cox1t.rds")
tga_mtlr=readRDS("savedresults/anz_ga_mtlrt.rds")
tga_coxboost=readRDS("savedresults/anz_ga_coxboostt.rds")
tlimma_mtlr=readRDS("savedresults/anz_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/anz_limma_coxboostt.rds")
tdeepsurv=readRDS("savedresults/anz_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/anz_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g7=plot_fun1(m)
g8=plot_fun2(t)
anzall=cbind.data.frame(m,t,m1,t1)
anzall$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#us
cox1=readRDS("savedresults/us_cox1m.rds")
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/us_p_cox1m.rds")
pcox2=readRDS("savedresults/us_p_cox2m.rds")
pcox3=readRDS("savedresults/us_p_cox3m.rds")
rsf1=readRDS("savedresults/us_rsf1m.rds")
rsf2=readRDS("savedresults/us_rsf2m.rds")
mtlr1=readRDS("savedresults/us_mtlrm.rds")
dnnsurv1=NULL
coxboost=readRDS("savedresults/us_coxboostm.rds")
survivalsvm=NULL
ga_cox=NULL
ga_mtlr=NULL
ga_coxboost=NULL
limma_mtlr=readRDS("savedresults/us_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/us_limma_coxboostm.rds")
deepsurv=readRDS("savedresults/us_deepsurvm.rds")
deephit=readRDS("savedresults/us_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=readRDS("savedresults/us_cox1t.rds")
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/us_p_cox1t.rds")*3600
tpcox2=readRDS("savedresults/us_p_cox2t.rds")
tpcox3=readRDS("savedresults/us_p_cox3t.rds")*3600
trsf1=readRDS("savedresults/us_rsf1t.rds")*3600
trsf2=readRDS("savedresults/us_rsf2t.rds")*60
tmtlr1=readRDS("savedresults/us_mtlrt.rds")*3600
tdnnsurv1=NULL
tcoxboost=readRDS("savedresults/us_coxboostt.rds")
tsurvivalsvm=NULL
tga_cox=NULL
tga_mtlr=NULL
tga_coxboost=NULL
tlimma_mtlr=readRDS("savedresults/us_limma_mtlrt.rds")*60
tlimma_coxboost=readRDS("savedresults/us_limma_coxboostt.rds")
tdeepsurv=readRDS("savedresults/us_deepsurvt.rds")*3600
tdeephit=readRDS("savedresults/us_deephitt.rds")*3600
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g9=plot_fun1(m)
g10=plot_fun2(t)
usall=cbind.data.frame(m,t,m1,t1)
usall$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#melanomaclinical

cox1=readRDS("savedresults/melanomaclinical_cox1m.rds")
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/melanomaclinical_p_cox1m.rds")
pcox2=readRDS("savedresults/melanomaclinical_p_cox2m.rds")
pcox3=readRDS("savedresults/melanomaclinical_p_cox3m.rds")
rsf1=readRDS("savedresults/melanomaclinical_rsf1m.rds")
rsf2=readRDS("savedresults/melanomaclinical_rsf2m.rds")
mtlr1=readRDS("savedresults/melanomaclinical_mtlrm.rds")
dnnsurv1=readRDS("savedresults/melanomaclinical_dnnsurvm.rds")
coxboost=readRDS("savedresults/melanomaclinica_coxboostm.rds")
survivalsvm=NULL
ga_cox=NULL
ga_mtlr=NULL
ga_coxboost=NULL
limma_mtlr=readRDS("savedresults/melanomaclinical_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/melanomaclinical_limma_coxboostm.rds")
deepsurv=readRDS("savedresults/melanomaclinical_deepsurvm.rds")
deephit=readRDS("savedresults/melanomaclinical_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=readRDS("savedresults/melanomaclinical_cox1t.rds")
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/melanomaclinical_p_cox1t.rds")
tpcox2=readRDS("savedresults/melanomaclinical_p_cox2t.rds")
tpcox3=readRDS("savedresults/melanomaclinical_p_cox3t.rds")
trsf1=readRDS("savedresults/melanomaclinical_rsf1t.rds")*60
trsf2=readRDS("savedresults/melanomaclinical_rsf2t.rds")
tmtlr1=readRDS("savedresults/melanomaclinical_mtlrt.rds")
tdnnsurv1=readRDS("savedresults/melanomaclinical_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/melanomaclinica_coxboostt.rds")
tsurvivalsvm=NULL
tga_cox=NULL
tga_mtlr=NULL
tga_coxboost=NULL
tlimma_mtlr=readRDS("savedresults/melanomaclinical_limma_mtlrt.rds")*60
tlimma_coxboost=readRDS("savedresults/melanomaclinical_limma_coxboostt.rds")
tdeepsurv=readRDS("savedresults/melanomaclinical_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/melanomaclinical_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)

m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g11=plot_fun1(m)
g12=plot_fun2(t)
melanomaclinicalall=cbind.data.frame(m,t,m1,t1)
melanomaclinicalall$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")

#melanomaitraq
cox1=NULL
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/melanomaitraqv2_p_cox1m.rds")
pcox2=NULL
pcox3=NULL
rsf1=readRDS("savedresults/melanomaitraqv2_rsf1m.rds")
rsf2=readRDS("savedresults/melanomaitraqv2_rsf2m.rds")
mtlr1=readRDS("savedresults/melanomaitraqv2_mtlrm.rds")
dnnsurv1=readRDS("savedresults/melanomaitraqv2_dnnsurvm.rds")
coxboost=readRDS("savedresults/melanomaitraqv2_coxboostm.rds")
ga_cox=readRDS("savedresults/itraqv2_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/melanomaitraqv2_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/melanomaitraqv2_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/melanomaitraqv2_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/melanomaitraqv2_limma_coxboostm.rds")
survivalsvm=readRDS("savedresults/itraqv2_survivalsvmm.rds")
deepsurv=readRDS("savedresults/melanomaitraqv2_deepsurvm.rds")
deephit=readRDS("savedresults/melanomaitraqv2_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=NULL
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/melanomaitraq_p_cox1t.rds")
tpcox2=NULL
tpcox3=NULL
trsf1=readRDS("savedresults/melanomaitraqv2_rsf1t.rds")
trsf2=readRDS("savedresults/melanomaitraqv2_rsf2t.rds")
tmtlr1=readRDS("savedresults/melanomaitraqv2_mtlrt.rds")*60
tdnnsurv1=readRDS("savedresults/melanomaitraqv2_dnnsurvt.rds")
tcoxboost=readRDS("savedresults/melanomaitraqv2_coxboostt.rds")
tga_cox=readRDS("savedresults/itraqv2_ga_cox1t.rds")
tga_mtlr=readRDS("savedresults/melanomaitraqv2_ga_mtlrt.rds")
tga_coxboost=readRDS("savedresults/melanomaitraqv2_ga_coxboostt.rds")
tlimma_mtlr=readRDS("savedresults/melanomaitraqv2_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/melanomaitraqv2_limma_coxboostt.rds")
tsurvivalsvm=readRDS("savedresults/itraqv2_survivalsvmt.rds")
tdeepsurv=readRDS("savedresults/melanomaitraqv2_deepsurvt.rds")
tdeephit=readRDS("savedresults/melanomaitraqv2_deephitt.rds")
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)

m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g13=plot_fun1(m)
g14=plot_fun2(t)
melanomaitraqall=cbind.data.frame(m,t,m1,t1)
melanomaitraqall$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")

#melanomanano
cox1=NULL
cox2=readRDS("savedresults/melanomananov3_bw_cox1m.rds")
cox3=readRDS("savedresults/melanomananov3_bw_cox2m.rds")
cox4=readRDS("savedresults/melanomananov3_bw_cox3m.rds")
pcox1=readRDS("savedresults/melanomananov3_p_cox1m.rds")
pcox2=readRDS("savedresults/melanomananov3_p_cox2m.rds")
pcox3=readRDS("savedresults/melanomananov3_p_cox3m.rds")
rsf1=readRDS("savedresults/melanomanano_rsf1m.rds")
rsf2=readRDS("savedresults/melanomanano_rsf2m.rds")
mtlr1=readRDS("savedresults/melanomananov3_mtlrm.rds")
dnnsurv1=readRDS("savedresults/melanomananov3_dnnsurvm.rds")
coxboost=readRDS("savedresults/melanomananov3_coxboostm.rds")
ga_cox=readRDS("savedresults/melanomananov3_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/melanomananov3_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/melanomananov3_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/melanomananov3_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/melanomananov3_limma_coxboostm.rds")
survivalsvm=NULL
deepsurv=readRDS("savedresults/melanomananov3_deepsurvm.rds")
deephit=readRDS("savedresults/melanomananov3_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=NULL
tcox2=readRDS("savedresults/melanomananov3_bw_cox1t.rds")
tcox3=readRDS("savedresults/melanomananov3_bw_cox2t.rds")
tcox4=readRDS("savedresults/melanomananov3_bw_cox3t.rds")
tpcox1=readRDS("savedresults/melanomananov3_p_cox1t.rds")
tpcox2=readRDS("savedresults/melanomananov3_p_cox2t.rds")
tpcox3=readRDS("savedresults/melanomananov3_p_cox3t.rds")
trsf1=readRDS("savedresults/melanomanano_rsf1t.rds")
trsf2=readRDS("savedresults/melanomanano_rsf2t.rds")
tmtlr1=readRDS("savedresults/melanomananov3_mtlrt.rds")
tdnnsurv1=readRDS("savedresults/melanomananov3_dnnsurvt.rds")
tcoxboost=readRDS("savedresults/melanomananov3_coxboostt.rds")
tga_cox=readRDS("savedresults/melanomananov3_ga_cox1t.rds")
tga_mtlr=readRDS("savedresults/melanomananov3_ga_mtlrt.rds")
tga_coxboost=readRDS("savedresults/melanomananov3_ga_coxboostt.rds")
tlimma_mtlr=readRDS("savedresults/melanomananov3_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/melanomananov3_limma_coxboostt.rds")
tsurvivalsvm=NULL
tdeepsurv=readRDS("savedresults/melanomananov3_deepsurvt.rds")
tdeephit=readRDS("savedresults/melanomananov3_deephitt.rds")
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g17=plot_fun1(m)
g18=plot_fun2(t)
melanomananoall=cbind.data.frame(m,t,m1,t1)
melanomananoall$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#gse1
cox1=NULL
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/gse1_p_cox1m.rds")
pcox2=readRDS("savedresults/gse1_p_cox2m.rds")
pcox3=readRDS("savedresults/gse1_p_cox3m.rds")
rsf1=readRDS("savedresults/gse1_rsf1m.rds")
rsf2=readRDS("savedresults/gse1_rsf2m.rds")
mtlr1=NULL
dnnsurv1=NULL
coxboost=readRDS("savedresults/gse1_coxboostm.rds")
ga_cox=readRDS("savedresults/gse1_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/gse1_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/gse1_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/gse1_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/gse1_limma_coxboostm.rds")
survivalsvm=readRDS("savedresults/gse1_survivalsvmm.rds")
deepsurv=readRDS("savedresults/gse1_deepsurvm.rds")
deephit=readRDS("savedresults/gse1_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=NULL
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/gse1_p_cox1t.rds")*60
tpcox2=readRDS("savedresults/gse1_p_cox2t.rds")*60
tpcox3=readRDS("savedresults/gse1_p_cox3t.rds")*60
trsf1=readRDS("savedresults/gse1_rsf1t.rds")*60
trsf2=readRDS("savedresults/gse1_rsf2t.rds")*60
tmtlr1=NULL
tdnnsurv1=NULL
tcoxboost=readRDS("savedresults/gse1_coxboostt.rds")*60
tga_cox=readRDS("savedresults/gse1_ga_cox1t.rds")*60
tga_mtlr=readRDS("savedresults/gse1_ga_mtlrt.rds")*60
tga_coxboost=readRDS("savedresults/gse1_ga_coxboostt.rds")*60
tlimma_mtlr=readRDS("savedresults/gse1_limma_mtlrt.rds")*60
tlimma_coxboost=readRDS("savedresults/gse1_limma_coxboostt.rds")
tsurvivalsvm=readRDS("savedresults/gse1_survivalsvmt.rds")
tdeepsurv=readRDS("savedresults/melanomananov3_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/melanomananov3_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g19=plot_fun1(m)
g20=plot_fun2(t)
gse1all=cbind.data.frame(m,t,m1,t1)
gse1all$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#gse2
cox1=NULL
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/gse4_p_cox1m.rds")
pcox2=readRDS("savedresults/gse4_p_cox2m.rds")
pcox3=readRDS("savedresults/gse4_p_cox3m.rds")
rsf1=readRDS("savedresults/gse4_rsf1m.rds")
rsf2=readRDS("savedresults/gse4_rsf2m.rds")
mtlr1=NULL
dnnsurv1=NULL
coxboost=readRDS("savedresults/gse4_coxboostm.rds")
ga_cox=readRDS("savedresults/gse4_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/gse4_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/gse4_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/gse4_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/gse4_limma_coxboostm.rds")
survivalsvm=readRDS("savedresults/gse4_survivalsvmm.rds")
deepsurv=NULL
deephit=NULL
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=NULL
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/gse4_p_cox1t.rds")*60
tpcox2=readRDS("savedresults/gse4_p_cox2t.rds")*60
tpcox3=readRDS("savedresults/gse4_p_cox3t.rds")*60
trsf1=readRDS("savedresults/gse4_rsf1t.rds")*60
trsf2=readRDS("savedresults/gse4_rsf2t.rds")*60
tmtlr1=NULL
tdnnsurv1=NULL
tcoxboost=readRDS("savedresults/gse4_coxboostt.rds")
tga_cox=readRDS("savedresults/gse4_ga_cox1t.rds")*60
tga_mtlr=readRDS("savedresults/gse4_ga_mtlrt.rds")*60
tga_coxboost=readRDS("savedresults/gse4_ga_coxboostt.rds")*60
tlimma_mtlr=readRDS("savedresults/gse4_limma_mtlrt.rds")*60
tlimma_coxboost=readRDS("savedresults/gse4_limma_coxboostt.rds")
tsurvivalsvm=readRDS("savedresults/gse4_survivalsvmt.rds")
tdeepsurv=NULL
tdeephit=NULL
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g21=plot_fun1(m)
g22=plot_fun2(t)
gse2all=cbind.data.frame(m,t,m1,t1)
gse2all$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#ngene1
cox1=NULL
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/ngse1_p_cox1m.rds")
pcox2=readRDS("savedresults/ngse1_p_cox2m.rds")
pcox3=readRDS("savedresults/ngse1_p_cox3m.rds")
rsf1=readRDS("savedresults/ngse1_rsf1m.rds")
rsf2=readRDS("savedresults/ngse1_rsf2m.rds")
mtlr1=readRDS("savedresults/ngse1_mtlrm.rds")
dnnsurv1=readRDS("savedresults/ngse1_dnnsurvm.rds")
coxboost=readRDS("savedresults/ngse1_coxboostm.rds")
ga_cox=readRDS("savedresults/ngse1_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/ngse1_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/ngse1_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/ngse1_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/ngse1_limma_coxboostm.rds")
survivalsvm=readRDS("savedresults/ngse1_survivalsvmm.rds")
deepsurv=NULL
deephit=NULL
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=NULL
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/ngse1_p_cox1t.rds")
tpcox2=readRDS("savedresults/ngse1_p_cox2t.rds")
tpcox3=readRDS("savedresults/ngse1_p_cox3t.rds")
trsf1=readRDS("savedresults/ngse1_rsf1t.rds")
trsf2=readRDS("savedresults/ngse1_rsf2t.rds")
tmtlr1=readRDS("savedresults/ngse1_mtlrt.rds")*60
tdnnsurv1=readRDS("savedresults/ngse1_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/ngse1_coxboostt.rds")
tga_cox=readRDS("savedresults/ngse1_ga_cox1t.rds")*60
tga_mtlr=readRDS("savedresults/ngse1_ga_mtlrt.rds")*60
tga_coxboost=readRDS("savedresults/ngse1_ga_coxboostt.rds")*60
tlimma_mtlr=readRDS("savedresults/ngse1_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/ngse1_limma_coxboostt.rds")
tsurvivalsvm=readRDS("savedresults/ngse1_survivalsvmt.rds")
tdeepsurv=NULL
tdeephit=NULL
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g23=plot_fun1(m)
g24=plot_fun2(t)
ngene1all=cbind.data.frame(m,t,m1,t1)
ngene1all$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#ngene2
cox1=NULL
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/ngse2_p_cox1m.rds")
pcox2=readRDS("savedresults/ngse2_p_cox2m.rds")
pcox3=readRDS("savedresults/ngse2_p_cox3m.rds")
rsf1=readRDS("savedresults/ngse2_rsf1m.rds")
rsf2=readRDS("savedresults/ngse2_rsf2m.rds")
mtlr1=readRDS("savedresults/ngse2_mtlrm.rds")
dnnsurv1=readRDS("savedresults/ngse2_dnnsurvm.rds")
coxboost=readRDS("savedresults/ngse2_coxboostm.rds")
ga_cox=readRDS("savedresults/ngse2_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/ngse2_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/ngse2_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/ngse2_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/ngse2_limma_coxboostm.rds")
survivalsvm=readRDS("savedresults/ngse2_survivalsvmm.rds")
deepsurv=readRDS("savedresults/ngse2_deepsurvm.rds")
deephit=readRDS("savedresults/ngse2_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=NULL
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/ngse2_p_cox1t.rds")*60
tpcox2=readRDS("savedresults/ngse2_p_cox2t.rds")*60
tpcox3=readRDS("savedresults/ngse2_p_cox3t.rds")*60
trsf1=readRDS("savedresults/ngse2_rsf1t.rds")*60
trsf2=readRDS("savedresults/ngse2_rsf2t.rds")*60
tmtlr1=readRDS("savedresults/ngse2_mtlrt.rds")*3600
tdnnsurv1=readRDS("savedresults/ngse2_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/ngse2_coxboostt.rds")
tga_cox=readRDS("savedresults/ngse2_ga_cox1t.rds")*60
tga_mtlr=readRDS("savedresults/ngse2_ga_mtlrt.rds")*60
tga_coxboost=readRDS("savedresults/ngse2_ga_coxboostt.rds")*60
tlimma_mtlr=readRDS("savedresults/ngse2_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/ngse2_limma_coxboostt.rds")
tsurvivalsvm=readRDS("savedresults/ngse2_survivalsvmt.rds")*60
tdeepsurv=readRDS("savedresults/ngse2_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/ngse2_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)

m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g25=plot_fun1(m)
g26=plot_fun2(t)
ngene2all=cbind.data.frame(m,t,m1,t1)
ngene2all$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#ngene3
cox1=NULL
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/ngse3_p_cox1m.rds")
pcox2=readRDS("savedresults/ngse3_p_cox2m.rds")
pcox3=readRDS("savedresults/ngse3_p_cox3m.rds")
rsf1=readRDS("savedresults/ngse3_rsf1m.rds")
rsf2=readRDS("savedresults/ngse3_rsf2m.rds")
mtlr1=readRDS("savedresults/ngse3_mtlrm.rds")
dnnsurv1=readRDS("savedresults/ngse3_dnnsurvm.rds")
coxboost=readRDS("savedresults/ngse3_coxboostm.rds")
ga_cox=readRDS("savedresults/ngse3_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/ngse3_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/ngse3_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/ngse3_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/ngse3_limma_coxboostm.rds")
survivalsvm=NULL
deepsurv=readRDS("savedresults/ngse3_deepsurvm.rds")
deephit=readRDS("savedresults/ngse3_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=NULL
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/ngse3_p_cox1t.rds")
tpcox2=readRDS("savedresults/ngse3_p_cox2t.rds")*60
tpcox3=readRDS("savedresults/ngse3_p_cox3t.rds")
trsf1=readRDS("savedresults/ngse3_rsf1t.rds")*60
trsf2=readRDS("savedresults/ngse3_rsf2t.rds")*60
tmtlr1=readRDS("savedresults/ngse3_mtlrt.rds")*3600
tdnnsurv1=readRDS("savedresults/ngse3_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/ngse3_coxboostt.rds")
tga_cox=readRDS("savedresults/ngse3_ga_cox1t.rds")*60
tga_mtlr=readRDS("savedresults/ngse3_ga_mtlrt.rds")*60
tga_coxboost=readRDS("savedresults/ngse3_ga_coxboostt.rds")*60
tlimma_mtlr=readRDS("savedresults/ngse3_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/ngse3_limma_coxboostt.rds")
tsurvivalsvm=NULL
tdeepsurv=readRDS("savedresults/ngse3_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/ngse3_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g27=plot_fun1(m)
g28=plot_fun2(t)
ngene3all=cbind.data.frame(m,t,m1,t1)
ngene3all$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#ngene4
cox1=NULL
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/ngse4_p_cox1m.rds")
pcox2=readRDS("savedresults/ngse4_p_cox2m.rds")
pcox3=readRDS("savedresults/ngse4_p_cox3m.rds")
rsf1=readRDS("savedresults/ngse4_rsf1m.rds")
rsf2=readRDS("savedresults/ngse4_rsf2m.rds")
mtlr1=readRDS("savedresults/ngse4_mtlrm.rds")
dnnsurv1=readRDS("savedresults/ngse4_dnnsurvm.rds")
coxboost=readRDS("savedresults/ngse4_coxboostm.rds")
ga_cox=readRDS("savedresults/ngse4_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/ngse4_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/ngse4_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/ngse4_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/ngse4_limma_coxboostm.rds")
survivalsvm=readRDS("savedresults/ngse4_survivalsvmm.rds")
deepsurv=readRDS("savedresults/ngse4_deepsurvm.rds")
deephit=readRDS("savedresults/ngse4_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=NULL
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/ngse4_p_cox1t.rds")
tpcox2=readRDS("savedresults/ngse4_p_cox2t.rds")
tpcox3=readRDS("savedresults/ngse4_p_cox3t.rds")
trsf1=readRDS("savedresults/ngse4_rsf1t.rds")*60
trsf2=readRDS("savedresults/ngse4_rsf2t.rds")*60
tmtlr1=readRDS("savedresults/ngse4_mtlrt.rds")*3600
tdnnsurv1=readRDS("savedresults/ngse4_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/ngse4_coxboostt.rds")
tga_cox=readRDS("savedresults/ngse4_ga_cox1t.rds")*60
tga_mtlr=readRDS("savedresults/ngse4_ga_mtlrt.rds")*60
tga_coxboost=readRDS("savedresults/ngse4_ga_coxboostt.rds")*60
tlimma_mtlr=readRDS("savedresults/ngse4_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/ngse4_limma_coxboostt.rds")
tsurvivalsvm=readRDS("savedresults/ngse4_survivalsvmt.rds")
tdeepsurv=readRDS("savedresults/ngse4_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/ngse4_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)

m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g29=plot_fun1(m)
g30=plot_fun2(t)
ngene4all=cbind.data.frame(m,t,m1,t1)
ngene4all$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#ngene5
cox1=NULL
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/ngse5_p_cox1m.rds")
pcox2=readRDS("savedresults/ngse5_p_cox2m.rds")
pcox3=readRDS("savedresults/ngse5_p_cox3m.rds")
rsf1=readRDS("savedresults/ngse5_rsf1m.rds")
rsf2=readRDS("savedresults/ngse5_rsf2m.rds")
mtlr1=readRDS("savedresults/ngse5_mtlrm.rds")
dnnsurv1=readRDS("savedresults/ngse5_dnnsurvm.rds")
coxboost=readRDS("savedresults/ngse5_coxboostm.rds")
ga_cox=readRDS("savedresults/ngse5_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/ngse5_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/ngse5_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/ngse5_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/ngse5_limma_coxboostm.rds")
survivalsvm=readRDS("savedresults/ngse5_survivalsvmm.rds")
deepsurv=readRDS("savedresults/ngse5_deepsurvm.rds")
deephit=readRDS("savedresults/ngse5_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=NULL
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/ngse5_p_cox1t.rds")
tpcox2=readRDS("savedresults/ngse5_p_cox2t.rds")
tpcox3=readRDS("savedresults/ngse5_p_cox3t.rds")
trsf1=readRDS("savedresults/ngse5_rsf1t.rds")*60
trsf2=readRDS("savedresults/ngse5_rsf2t.rds")*60
tmtlr1=readRDS("savedresults/ngse5_mtlrt.rds")*60
tdnnsurv1=readRDS("savedresults/ngse5_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/ngse5_coxboostt.rds")
tga_cox=readRDS("savedresults/ngse5_ga_cox1t.rds")*60
tga_mtlr=readRDS("savedresults/ngse5_ga_mtlrt.rds")*60
tga_coxboost=readRDS("savedresults/ngse5_ga_coxboostt.rds")*60
tlimma_mtlr=readRDS("savedresults/ngse5_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/ngse5_limma_coxboostt.rds")
tsurvivalsvm=readRDS("savedresults/ngse5_survivalsvmt.rds")
tdeepsurv=readRDS("savedresults/ngse5_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/ngse5_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g31=plot_fun1(m)
g32=plot_fun2(t)
ngene5all=cbind.data.frame(m,t,m1,t1)
ngene5all$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")
#ngene6
cox1=NULL
cox2=NULL
cox3=NULL
cox4=NULL
pcox1=readRDS("savedresults/ngse6_p_cox1m.rds")
pcox2=readRDS("savedresults/ngse6_p_cox2m.rds")
pcox3=readRDS("savedresults/ngse6_p_cox3m.rds")
rsf1=readRDS("savedresults/ngse6_rsf1m.rds")
rsf2=readRDS("savedresults/ngse6_rsf2m.rds")
mtlr1=readRDS("savedresults/ngse6_mtlrm.rds")
dnnsurv1=readRDS("savedresults/ngse6_dnnsurvm.rds")
coxboost=readRDS("savedresults/ngse6_coxboostm.rds")
ga_cox=readRDS("savedresults/ngse6_ga_cox1m.rds")
ga_mtlr=readRDS("savedresults/ngse6_ga_mtlrm.rds")
ga_coxboost=readRDS("savedresults/ngse6_ga_coxboostm.rds")
limma_mtlr=readRDS("savedresults/ngse6_limma_mtlrm.rds")
limma_coxboost=readRDS("savedresults/ngse6_limma_coxboostm.rds")
survivalsvm=readRDS("savedresults/ngse6_survivalsvmm.rds")
deepsurv=readRDS("savedresults/ngse6_deepsurvm.rds")
deephit=readRDS("savedresults/ngse6_deephitm.rds")
model_list1=list(cox1,cox2,cox3,cox4,pcox1,pcox2,pcox3,rsf1,rsf2,mtlr1,dnnsurv1,coxboost,ga_cox,ga_mtlr,ga_coxboost,limma_mtlr,limma_coxboost,survivalsvm,deepsurv,deephit)

tcox1=NULL
tcox2=NULL
tcox3=NULL
tcox4=NULL
tpcox1=readRDS("savedresults/ngse6_p_cox1t.rds")*60
tpcox2=readRDS("savedresults/ngse6_p_cox2t.rds")*60
tpcox3=readRDS("savedresults/ngse6_p_cox3t.rds")*60
trsf1=readRDS("savedresults/ngse6_rsf1t.rds")*60
trsf2=readRDS("savedresults/ngse6_rsf2t.rds")*60
tmtlr1=readRDS("savedresults/ngse6_mtlrt.rds")*3600
tdnnsurv1=readRDS("savedresults/ngse6_dnnsurvt.rds")*60
tcoxboost=readRDS("savedresults/ngse6_coxboostt.rds")
tga_cox=readRDS("savedresults/ngse6_ga_cox1t.rds")*60
tga_mtlr=readRDS("savedresults/ngse6_ga_mtlrt.rds")*60
tga_coxboost=readRDS("savedresults/ngse6_ga_coxboostt.rds")
tlimma_mtlr=readRDS("savedresults/ngse6_limma_mtlrt.rds")
tlimma_coxboost=readRDS("savedresults/ngse6_limma_coxboostt.rds")
tsurvivalsvm=readRDS("savedresults/ngse6_survivalsvmt.rds")*60
tdeepsurv=readRDS("savedresults/ngse6_deepsurvt.rds")*60
tdeephit=readRDS("savedresults/ngse6_deephitt.rds")*60
model_list2=list(tcox1,tcox2,tcox3,tcox4,tpcox1,tpcox2,tpcox3,trsf1,trsf2,tmtlr1,tdnnsurv1,tcoxboost,tga_cox,tga_mtlr,tga_coxboost,tlimma_mtlr,tlimma_coxboost,tsurvivalsvm,tdeepsurv,tdeephit)
m=memory_fun(model_list1)
t=time_fun(model_list2)
m1=as.vector(ifelse(m>10,"large",ifelse(m<5,"small","medium")))
t1=as.vector(ifelse(t<100,"fast",ifelse(t>1000,"slow","ok")))
g33=plot_fun1(m)
g34=plot_fun2(t)
ngene6all=cbind.data.frame(m,t,m1,t1)
ngene6all$methods=c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","rsf1","rsf2","mtlr1","dnnsurv1","coxboost","gacox","gamtlr","gacoxboost","limmamtlr","limmacoxboost","survivalsvm","deepsurv","deephit")

labelss<-c("pbc","veteran","lung","anz","us","melanoma_clinical","melanoma_itraq","melanoma_swath","melanoma_nano","gse1","gse2","ngene1","ngene2","ngene3","ngene4","ngene5","ngene6")
ggarrange(g1,g3,g5,g7,g9,g11,g13,g15,g17,g19,g21,g23,g25,g27,g29,g31,g33,labels = labelss,ncol = 3, nrow = 6)

ggarrange(g2,g4,g6,g8,g10,g12,g14,g16,g18,g20,g22,g24,g24,g26,g28,g30,g32,g34,labels = labelss,ncol = 3, nrow = 6)

library(DT)
data_list2=list(pbcall,veteranall,lungall,anzall,usall,melanomaclinicalall,melanomaitraqall,melanomananoall,gse1all,gse2all,ngene1all,ngene2all,ngene3all,ngene4all,ngene5all,ngene6all)
datafull2=bind_rows(data_list2, .id = "datasets")
datafull2$datasets=mapvalues(datafull2$datasets, from = seq(1,16,1), to = c("pbc","veteran","lung","anz","us","melanomaclinical","melanomaitraq","melanomanano","gse1","gse2","ngene1","ngene2","mgene3","ngene4","ngene5","ngene6"))
datafull2$m2=ifelse(datafull2$m>3,"large",ifelse(datafull2$m<1,"small","medium"))
datafull2$t2=ifelse(datafull2$t<50,"fast",ifelse(datafull2$t>1000,"slow","ok"))
DT::datatable(datafull2)

na.omit(datafull2[datafull2$m2=="large"&datafull2$t2=="slow",])

##             datasets      m         t     m1   t1   methods    m2   t2
## 68               anz    6.6  1803.785 medium slow      rsf1 large slow
## 71               anz   22.7  1236.030  large slow  dnnsurv1 large slow
## 85                us   20.4 21670.952  large slow     pcox1 large slow
## 87                us    7.6  8868.669 medium slow     pcox3 large slow
## 88                us    4.8  4113.810  small slow      rsf1 large slow
## 90                us    8.6  9217.211 medium slow     mtlr1 large slow
## 99                us   18.0  3917.326  large slow  deepsurv large slow
## 108 melanomaclinical    5.2  2032.390 medium slow      rsf1 large slow
## 168             gse1    4.9  2956.533  small slow      rsf1 large slow
## 176             gse1 3393.8  1025.182  large slow limmamtlr large slow
## 188             gse2    4.9  1932.757  small slow      rsf1 large slow
## 230           ngene2    3.5  9644.277  small slow     mtlr1 large slow
## 270           ngene4    3.1  3922.781  small slow     mtlr1 large slow
## 290           ngene5    4.0  2270.606  small slow     mtlr1 large slow

#datafull2[datafull2$m1=="large"&datafull2$t1=="ok",]
#datafull2[datafull2$m1=="medium"&datafull2$t1=="slow",]

5 all summary

data_list2=list(pbcall,veteranall,lungall,anzall,usall,melanomaclinicalall,melanomaitraqall,melanomananoall,gse1all,gse2all,ngene1all,ngene2all,ngene3all,ngene4all,ngene5all,ngene6all)
datafull2=bind_rows(data_list2, .id = "datasets")
datafull2$datasets=mapvalues(datafull2$datasets, from = seq(1,16,1), to = c("pbc","veteran","lung","anz","us","melanomaclinical","melanomaitraq","melanomanano","gse1","gse2","ngene1","ngene2","mgene3","ngene4","ngene5","ngene6"))
timememorydt=datafull2%>% group_by(methods) %>%dplyr::summarize(Mean_time = mean(t, na.rm=TRUE),Mean_memory=mean(m,na.rm = TRUE))

datafull3=datafull
datafull3$datasets=mapvalues(datafull3$datasets, from = seq(1,16,1), to = c("anz","us","veteran","lung","pbc","melanoma_clinical","melanomaitraq","melanomanano","gse1","gse2","ngene1","ngene2","mgene3","ngene4","ngene5","ngene6"))
predictiondt=datafull3%>% group_by(model,variable) %>%dplyr::summarize(Mean_value = mean(value, na.rm=TRUE), SD_value=sd(value,na.rm = TRUE))
predictiondt2=predictiondt[predictiondt$variable %in% c("hc", "unoc", "bs1","auc1", "auc5", "auc10", "auc15"),]
predictiondt3 <- tidyr::spread(predictiondt2[,1:3], variable, Mean_value)
predictiondt4=tidyr::spread(predictiondt2[,c(1,2,4)], variable, SD_value)


allsummarydt=cbind.data.frame(timememorydt,predictiondt3[,2:8],predictiondt4[,2:8])

colnames(allsummarydt)=c("methods","Mean_time","Mean_memory","Mean_hc","Mean_unoc","Mean_bs","Mean_auc1","Mean_auc5","Mean_auc10","Mean_auc15","SD_hc","SD_unoc","SD_bs","SD_auc1","SD_auc5","SD_auc10","SD_auc15")

#allsummarydt[allsummarydt$methods=="survivalsvm","Mean_bs"]==10*5#set to a large number because bs cant be calculated for survivalsvm actually
allsummarydt2=select(allsummarydt, -methods) %>% mutate_all(funs(dense_rank(desc(.))))
allsummarydt3=select(allsummarydt, -methods) %>% mutate_all(funs(rank(.)))
allsummarydt4=cbind.data.frame(allsummarydt2[,c(3,4,6,7,8,9)],allsummarydt3[,c(1,2,5,10:16)])
rownames(allsummarydt4)=allsummarydt$methods


nb.cols <- 20
mycolors <- colorRampPalette(brewer.pal(8, "RdBu"))(nb.cols)

mat=as.matrix(allsummarydt4)

# my_group <- c("1","1","2","3","3","3","3","4","4","5","5","5","5","5","5","5")
# ha=HeatmapAnnotation(categories=my_group,col = list(categories = c("1" = "red", "4"="orange","2" = "green", "3" = "blue","5"="pink")))
# Heatmap(mat, name = "rank", col = mycolors,rect_gp = gpar(col = "white", lwd = 2),row_dend_reorder = FALSE,cluster_columns = FALSE,top_annotation = ha)


#my_group <- c("1","1","2","3","3","3","3","4","4","5","5","5","5","5","5","5")
my_group <- c("1","1","3","3","3","3","4","4","2","5","5","5","5","5","5","5")
ha=HeatmapAnnotation(categories=my_group,col = list(categories = c("1" = "red", "4"="orange","2" = "seagreen", "3" = "blue","5"="pink")))
Heatmap(mat, name = "rank", col = mycolors,rect_gp = gpar(col = "white", lwd = 2),cluster_columns = FALSE,top_annotation = ha,row_order = c("cox1","cox2","cox3","cox4","pcox1","pcox2","pcox3","coxboost","gacox","gacoxboost","limmacoxboost","mtlr1","gamtlr","limmamtlr","rsf1","rsf2","survivalsvm","deephit","deepsurv","dnnsurv1"),column_order = c("Mean_hc","Mean_unoc","Mean_auc1","Mean_auc5","Mean_auc10","Mean_auc15","Mean_bs","Mean_time","Mean_memory","SD_hc","SD_unoc","SD_bs","SD_auc1","SD_auc5","SD_auc10","SD_auc15"))

# nb.cols <- 18
# mycolors <- colorRampPalette(brewer.pal(8, "Blues"))(nb.cols)
# Heatmap(matrix(rnorm(6*18,0,1), ncol =18 ),col=mycolors)
# 
# nb.cols <- 18
# mycolors <- colorRampPalette(brewer.pal(8, "Set3"))(nb.cols)
# Heatmap(matrix(rnorm(6*18,0,1), ncol =18 ),col=mycolors)
# 
# nb.cols <- 18
# mycolors <- colorRampPalette(brewer.pal(8, "Set2"))(nb.cols)
# Heatmap(matrix(rnorm(6*18,0,1), ncol =18 ),col=mycolors)

datafull3=datafull
datafull3$datasets=mapvalues(datafull3$datasets, from = seq(1,16,1), to = c("anz","us","veteran","lung","pbc","melanoma_clinical","melanomaitraq","melanomanano","gse1","gse2","ngene1","ngene2","mgene3","ngene4","ngene5","ngene6"))

clinical_part=datafull3[datafull3$datasets %in% c("anz","us","veteran","lung","pbc","melanoma_clinical"),]
omics_part=datafull3[datafull3$datasets %in% c("melanomaitraq","melanomanano","gse1","gse2","ngene1","ngene2","mgene3","ngene4","ngene5","ngene6"),]

c_part=clinical_part%>% group_by(model,variable) %>%dplyr::summarize( SD_value=sd(value,na.rm = TRUE))
o_part=omics_part%>% group_by(model,variable) %>%dplyr::summarize( SD_value=sd(value,na.rm = TRUE))

c_part1=tidyr::spread(c_part, variable, SD_value)
o_part1=tidyr::spread(o_part, variable, SD_value)

c_part1=as.data.frame(c_part1[,c(1,2,26)])
c_part2=as.matrix(c_part1[,-1])
rownames(c_part2)=c_part1$model
nb.cols <- 18
mycolors <- colorRampPalette(brewer.pal(8, "RdBu"))(nb.cols)
mat=c_part2
Heatmap(t(mat), name = "SD", col=colorRamp2(c(0.05, 0.3), c("white", "red")),cluster_rows = FALSE,cluster_columns=FALSE,rect_gp = gpar(col = "white", lwd = 2),width = ncol(mat)*unit(5, "cm"), 
    height = nrow(mat)*unit(0.1, "cm"))

o_part1=as.data.frame(o_part1[,c(1,2,26)])
o_part2=as.matrix(o_part1[,-1])
rownames(o_part2)=o_part1$model
mat=o_part2
Heatmap(t(mat), name = "SD", col = colorRamp2(c(0.05, 0.3), c( "white", "red")),cluster_rows = FALSE,cluster_columns=FALSE,rect_gp = gpar(col = "white", lwd = 2),width = ncol(mat)*unit(5, "cm"), 
    height = nrow(mat)*unit(0.1, "cm"))